Routines for importing/exporting messages from/to XML message - catalog files. The key procedures in this library are + catalog files. Every OpenACS package has one message catalog file for + each locale (language and region) that its UI supports. Importing of messages means reading the messages + from XML catalog files and storing them in the database. Exporting of messages refers to the opposite process. + The key procedures in this library are:
-lang::catalog::export_messages_to_file - Export messages for a certain locale
- and package from the database to a given XML catalog file.lang::catalog::import_messages_from_file - Import messages for a certain
- locale and package from a given XML catalog file to the database.- This is free software distributed under the terms of the GNU Public - License. Full text of the license is available from the GNU Project: - http://www.fsf.org/copyleft/gpl.html +
- package_key - package_version - locale - charset - messages - An array with message keys as keys and the message texts as values. - descriptions - An array with message keys as keys and the descriptions as values. -- @author Peter Marklund (peter@collaboraid.biz) - @author Simon Carstensen (simon@collaboraid.biz) -} { +} { + set value [xml_node_get_attribute $element $attribute] - # Check arguments - if { [empty_string_p $catalog_file_contents] } { - error "lang::catalog::parse the catalog_file_contents arguments is the empty string" + if { [empty_string_p $value] } { + error "Required attribute \"$attribute\" missing from <[xml_node_get_name $element]>" } - # The names of xml tags and attributes - set MESSAGE_CATALOG_TAG "message_catalog" - set PACKAGE_KEY_ATTR "package_key" - set PACKAGE_VERSION_ATTR "package_version" - set LOCALE_ATTR "locale" - set CHARSET_ATTR "charset" - set MESSAGE_TAG "msg" - set DESCRIPTION_TAG "description" - set KEY_ATTR "key" - - # Initialize the array to return - array set msg_catalog_array {} - - # Parse the xml document - set tree [xml_parse -persist $catalog_file_contents] - - # Get the message catalog root node - set root_node [xml_doc_get_first_node $tree] - if { ![string equal [xml_node_get_name $root_node] ${MESSAGE_CATALOG_TAG}] } { - error "lang::catalog_parse: Could not find root node ${MESSAGE_CATALOG_TAG}" - } - - # Set the message catalog root level attributes - set msg_catalog_array(package_key) [get_required_xml_attribute $root_node ${PACKAGE_KEY_ATTR}] - set msg_catalog_array(package_version) [get_required_xml_attribute $root_node ${PACKAGE_VERSION_ATTR}] - set msg_catalog_array(locale) [get_required_xml_attribute $root_node ${LOCALE_ATTR}] - set msg_catalog_array(charset) [get_required_xml_attribute $root_node ${CHARSET_ATTR}] - - # Loop over the keys and message texts - set message_node_list [xml_node_get_children_by_name $root_node ${MESSAGE_TAG}] - array set key_text_array {} - foreach message_node $message_node_list { - set key [get_required_xml_attribute $message_node ${KEY_ATTR}] - set text [xml_node_get_content $message_node ] - set key_text_array($key) $text - } - - # Add the keys and the texts to the messages array - set msg_catalog_array(messages) [array get key_text_array] - - # Loop over the keys and descriptions - set description_node_list [xml_node_get_children_by_name $root_node ${DESCRIPTION_TAG}] - array set key_description_array {} - - foreach description_node $description_node_list { - set key [get_required_xml_attribute $description_node ${KEY_ATTR}] - set description [xml_node_get_content $description_node ] - set key_description_array($key) $description - } - - # Add the keys and the texts to the descriptions array - set msg_catalog_array(descriptions) [array get key_description_array] - - return [array get msg_catalog_array] + return $value } - ad_proc -private lang::catalog::get_required_xml_attribute { element attribute } { - Return the value of the given attribute and raise an error if the - value is missing or empty. - - @author Peter Marklund (peter@collaboraid.biz) - } { - set value [xml_node_get_attribute $element $attribute] - - if { [empty_string_p $value] } { - error "Required attribute \"$attribute\" missing from <[xml_node_get_name $element]>" - } - - return $value - } - - ad_proc -public lang::catalog::export_to_files { - {-locales ""} - } { - Export messages to catalog files for all enabled packages on the system. - - @param locales The locales to export. Defaults to all enabled locales that - a package has messages for in the database. - - @see lang::catalog::export_package_to_files - - @author Peter Marklund - } { - foreach package_key [apm_enabled_packages] { - lang::catalog::export_package_to_files -locales $locales $package_key - } - } - - ad_proc -public lang::catalog::export_package_to_files { - {-locales ""} - package_key -} { - Export all messages of the given package from the database to xml - catalog files. The messages for each locale are stored in its own file. - The catalog files are stored in the - directory /packages/package_key/catalog with a filename on the format - package_key.locale.charset.xml (i.e. dotlrn.en_US.iso-8859-1.xml). - - @param locales The locales to export. Defaults to all enabled locales that - the package has messages for in the database. - - @author Peter Marklund (peter@collaboraid.biz) - } { - # Loop over all locales that the package has messages in - # and write a catalog file for each such locale - db_foreach get_locales_for_package {} { - # If we are only exporting certain locales and this is not one of them - continue - if { ![empty_string_p $locales] && [lsearch -exact $locales $locale] == -1 } { - continue - } - - set charset [ad_locale charset $locale] - - # Get all messages in the current locale and put them in an array list - set messages_list [list] - set descriptions_list [list] - all_messages_for_package_and_locale $package_key $locale - template::util::multirow_foreach all_messages { - lappend messages_list @all_messages.message_key@ @all_messages.message@ - lappend descriptions_list @all_messages.message_key@ @all_messages.description@ - } - - # Write the messages and descriptions to the file - set catalog_file_path [get_catalog_file_path \ - -package_key $package_key \ - -locale $locale] - - export_messages_to_file -descriptions_list $descriptions_list \ - $catalog_file_path $messages_list - } -} - ad_proc -private lang::catalog::all_messages_for_package_and_locale { package_key locale } { Set a multirow with name all_messages locally in the callers scope with the columns message_key and message for all message keys that do @@ -276,7 +111,7 @@ set return_value 1 } else { # Catalog file with unknown prefix - ns_log Warning "lang::catalog::is_upgrade_backup_file - The file $file_path has unknown prefix $prefix" + ns_log Warning "The file $file_path has unknown prefix $prefix" set return_value 0 } } @@ -361,7 +196,50 @@ return $file_path } -ad_proc -public lang::catalog::export_messages_to_file { +ad_proc -private lang::catalog::get_catalog_files { package_key } { + Return the full paths of all message catalog files of the given package. + + @param package_key The key of the package to return catalog file paths for + + @return A list of catalog file paths + + @author Peter Marklund +} { + set catalog_paths [list] + + set catalog_dir [lang::catalog::package_catalog_dir $package_key] + foreach file_path [glob -nocomplain "$catalog_dir/*"] { + if { [apm_is_catalog_file $file_path] } { + lappend catalog_paths $file_path + } + } + + return $catalog_paths +} + +ad_proc -public lang::catalog::reset_upgrade_status_message_keys { package_key } { + Before a package upgrade the upgrade status of message keys is cleared + so that upgrade status always reflects the last upgrade. + + @author Peter Marklund +} { + db_dml reset_status {} +} + +ad_proc -private lang::catalog::system_package_version_name { package_key } { + Returns the version name of the highest version of the given + package_key in the system. +} { + return [db_string get_version_name {}] +} + +################## +# +# Exporting procs +# +################## + +ad_proc -private lang::catalog::export_to_file { {-descriptions_list ""} file_path messages_list @@ -453,37 +331,167 @@ ns_log Notice "Wrote $message_count messages to file $file_path with encoding $file_encoding" } -ad_proc -private lang::catalog::get_catalog_files { package_key } { - Return the full paths of the message catalog files of the given package. +ad_proc -public lang::catalog::export { + {-package_key {}} + {-locales {}} +} { + Exports I18N messages from the database to XML catalog files. By default exports messages + for all enabled packages and all enabled locales on the system. Can be restricted to export + only for a certain package and/or a list of locales. - @param package_key The key of the package to return catalog file paths for + @param package_key A key of a package to restrict the export to + @param locales A list of locales to restrict the export to - @return A list of catalog file paths - @author Peter Marklund } { - set catalog_paths [list] - - set catalog_dir [lang::catalog::package_catalog_dir $package_key] - foreach file_path [glob -nocomplain "$catalog_dir/*"] { - if { [apm_is_catalog_file $file_path] } { - lappend catalog_paths $file_path - } + if { ![empty_string_p $package_key] } { + set package_key_list $package_key + } else { + set package_key_list [apm_enabled_packages] } - return $catalog_paths + foreach package_key $package_key_list { + # Loop over all locales that the package has messages in + # and write a catalog file for each such locale + db_foreach get_locales_for_package {} { + # If we are only exporting certain locales and this is not one of them - continue + if { [llength $locales] > 0 && [lsearch -exact $locales $locale] == -1 } { + continue + } + + # Get messages and descriptions for the locale + set messages_list [list] + set descriptions_list [list] + all_messages_for_package_and_locale $package_key $locale + template::util::multirow_foreach all_messages { + lappend messages_list @all_messages.message_key@ @all_messages.message@ + lappend descriptions_list @all_messages.message_key@ @all_messages.description@ + } + + set catalog_file_path [get_catalog_file_path \ + -package_key $package_key \ + -locale $locale] + + export_to_file -descriptions_list $descriptions_list $catalog_file_path $messages_list + } + } } -ad_proc -public lang::catalog::reset_upgrade_status_message_keys { package_key } { - Before a package upgrade the upgrade status of message keys is cleared - so that upgrade status always reflects the last upgrade. +################## +# +# Importing procs +# +################## - @author Peter Marklund +ad_proc -private lang::catalog::read_file { catalog_filename } { + Returns the contents of the given catalog file as a string + reading the file with the charset given in the filename. + + @param catalog_file_name The full path of the catalog file to read. + The basename of the file should be on the form + package_key.locale.charset.ending where ending + is either cat or xml (i.e. dotlrn.en_US.iso-8859-1.xml + or dotlrn.en_US.iso-8859-1.cat). The cat ending + is for the deprecated tcl-based catalog files. + + @author Jeff Davis + @author Peter Marklund (peter@collaboraid.biz) } { - db_dml reset_status {} + if {![regexp {/([^/]*)\.([^/]*)\.(?:xml|cat)$} $catalog_filename match base msg_encoding]} { + ns_log Warning "Charset info missing in filename assuming $catalog_filename is iso-8859-1" + set msg_encoding iso-8859-1 + } + + set msg_encoding [default_charset_if_unsupported $msg_encoding] + + ns_log Notice "reading $catalog_filename in $msg_encoding" + set in [open $catalog_filename] + fconfigure $in -encoding [ns_encodingforcharset $msg_encoding] + set catalog_file_contents [read $in] + close $in + + return $catalog_file_contents } -ad_proc -public lang::catalog::import_messages_from_file { +ad_proc -private lang::catalog::parse { catalog_file_contents } { + Parse the given catalog file xml contents and return the data as + an array. The array will contain the following keys: + +
+ package_key + package_version + locale + charset + messages - An array with message keys as keys and the message texts as values. + descriptions - An array with message keys as keys and the descriptions as values. ++ + @author Peter Marklund (peter@collaboraid.biz) + @author Simon Carstensen (simon@collaboraid.biz) +} { + + # Check arguments + if { [empty_string_p $catalog_file_contents] } { + error "lang::catalog::parse the catalog_file_contents arguments is the empty string" + } + + # The names of xml tags and attributes + set MESSAGE_CATALOG_TAG "message_catalog" + set PACKAGE_KEY_ATTR "package_key" + set PACKAGE_VERSION_ATTR "package_version" + set LOCALE_ATTR "locale" + set CHARSET_ATTR "charset" + set MESSAGE_TAG "msg" + set DESCRIPTION_TAG "description" + set KEY_ATTR "key" + + # Initialize the array to return + array set msg_catalog_array {} + + # Parse the xml document + set tree [xml_parse -persist $catalog_file_contents] + + # Get the message catalog root node + set root_node [xml_doc_get_first_node $tree] + if { ![string equal [xml_node_get_name $root_node] ${MESSAGE_CATALOG_TAG}] } { + error "lang::catalog_parse: Could not find root node ${MESSAGE_CATALOG_TAG}" + } + + # Set the message catalog root level attributes + set msg_catalog_array(package_key) [get_required_xml_attribute $root_node ${PACKAGE_KEY_ATTR}] + set msg_catalog_array(package_version) [get_required_xml_attribute $root_node ${PACKAGE_VERSION_ATTR}] + set msg_catalog_array(locale) [get_required_xml_attribute $root_node ${LOCALE_ATTR}] + set msg_catalog_array(charset) [get_required_xml_attribute $root_node ${CHARSET_ATTR}] + + # Loop over the keys and message texts + set message_node_list [xml_node_get_children_by_name $root_node ${MESSAGE_TAG}] + array set key_text_array {} + foreach message_node $message_node_list { + set key [get_required_xml_attribute $message_node ${KEY_ATTR}] + set text [xml_node_get_content $message_node ] + set key_text_array($key) $text + } + + # Add the keys and the texts to the messages array + set msg_catalog_array(messages) [array get key_text_array] + + # Loop over the keys and descriptions + set description_node_list [xml_node_get_children_by_name $root_node ${DESCRIPTION_TAG}] + array set key_description_array {} + + foreach description_node $description_node_list { + set key [get_required_xml_attribute $description_node ${KEY_ATTR}] + set description [xml_node_get_content $description_node ] + set key_description_array($key) $description + } + + # Add the keys and the texts to the descriptions array + set msg_catalog_array(descriptions) [array get key_description_array] + + return [array get msg_catalog_array] +} + +ad_proc -private lang::catalog::import_from_file { file_path } {
@@ -533,7 +541,7 @@ # Compare xml package_key with file path package_key - abort if there is a mismatch if { ![string equal $package_key $catalog_array(package_key)] } { - error "lang::catalog::import_messages_from_file - the package_key $catalog_array(package_key) in the file $file_path does not match the package_key $package_key in the filesystem" + error "the package_key $catalog_array(package_key) in the file $file_path does not match the package_key $package_key in the filesystem" } # Figure out if we are upgrading @@ -635,76 +643,87 @@ } } -ad_proc -private lang::catalog::system_package_version_name { package_key } { - Returns the version name of the highest version of the given - package_key in the system. +ad_proc -public lang::catalog::import { + {-package_key {}} + {-locales {}} + {-cache:boolean} } { - return [db_string get_version_name {}] -} + Import messages from catalog files to the database. By default all messages + for enabled packages and enabled locales will be imported. Optionally, the import + can be restricted to a certain package and/or a list of locales. -ad_proc -public lang::catalog::import_from_files_for_locale { locale } { - Import messages for the given locale from catalog files in all enabled packages. + @param package_key Restrict the import to the package with this key + @param locales A list of locales to restrict the import to + @param cache Provide this switch if you want the proc to cache all the imported messages @author Peter Marklund } { - if { [lsearch [lang::system::get_locales] $locale] == -1 } { - error "lang::catalog::import_locale_from_files: Cannot import messages from files for locale $locale as it is not among the enabled locales ([lang::system::get_locales])" + if { ![empty_string_p $package_key] } { + set package_key_list $package_key + } else { + set package_key_list [apm_enabled_packages] } - foreach package_key [apm_enabled_packages] { + foreach package_key $package_key_list { # Skip the package if it has no catalog files at all if { ![file exists [package_catalog_dir $package_key]] } { continue } - import_from_files -restrict_to_locale $locale $package_key + set catalog_files [get_catalog_paths_for_import \ + -package_key $package_key \ + -locales $locales] + + # Issue a warning and exit if there are no catalog files + if { [empty_string_p $catalog_files] } { + ns_log Warning "No catalog files found for package $package_key" + continue + } + + foreach file_path $catalog_files { + # Use a catch so that parse failure of one file doesn't cause the import of all files to fail + if { [catch {import_messages_from_file $file_path} errMsg] } { + global errorInfo + + ns_log Error "The import of file $file_path failed, error message is:\n\n${errMsg}\n\nstack trace:\n\n$errorInfo\n\n" + } + } } + + if { $cache_p } { + lang::message::cache + } } -ad_proc -public lang::catalog::import_from_files { - {-restrict_to_locale ""} - package_key +ad_proc -private lang::catalog::get_catalog_paths_for_import { + {-package_key:required} + {-locales {}} } { - Import (load) all catalog files of a certain package. Catalog files - should be stored in the /packages/package_key/catalog directory - and have the ending .xml (i.e. /package/dotlrn/catalog/dotlrn.en_US.iso-8859-1.xml). - This procedure invokes lang::catalog::import_messages_from_file. + Return a list of file paths for the catalog files of the given package. Can + be restricted to only return files for certain locales. The list will + be sorted in an order appropriate for import to the database. - @param package_key The package key of the package to import catalog files for - @param restrict_to_locale Restrict importing to catalog files of the given locale + @param package_key The key of the package to get catalog file paths for + @param locales A list of locales to restrict the catalog files to - @author Peter Marklund (peter@collaboraid.biz) + @author Peter Marklund } { - # Check arguments - if { [empty_string_p $package_key] } { - error "lang::catalog::import_from_files - the package_key argument is the empty string" - } - - # Skip the package if it has no catalog files at all - if { ![file exists [package_catalog_dir $package_key]] } { - ns_log Notice "importing nothing as package $package_key has no catalog files" - return - } - # We always need to register en_US messages first as they create the keys set en_us_locale_list [list [list en_US [ad_locale charset en_US]]] set other_locales_list [db_list_of_lists locales_and_charsets { - select locale, - mime_charset + select locale from ad_locales where enabled_p = 't' and locale <> 'en_US' }] - set all_locales_list [concat $en_us_locale_list $other_locales_list] + set locales_list [concat $en_us_locale_list $other_locales_list] # Get all catalog files for enabled locales - set catalog_file_list [list] - foreach locale_list $all_locales_list { - set locale [lindex $locale_list 0] - set mime_charset [lindex $locale_list 1] + set catalog_files [list] + foreach locale $locales_list { # If we are only processing certain locales and this is not one of them - continue - if { ![empty_string_p $restrict_to_locale] && ![string equal $restrict_to_locale $locale]} { + if { [llength $locales] > 0 && [lsearch $locales $locale] == -1 } { continue } @@ -714,138 +733,30 @@ } set file_path [get_catalog_file_path \ - -package_key $package_key \ - -locale $locale] + -package_key $package_key \ + -locale $locale] if { [file exists $file_path] } { - lappend catalog_file_list $file_path + lappend catalog_files $file_path } else { - if { ![string equal $charset $mime_charset] } { - # File doesn't exist, but charset was unsupported and defaulted to something else (probably utf-8) - # For backward compatibility we check if the catalog has the mime charset in the filename even if it's - # unsupported - set alternate_file_path [get_catalog_file_path \ - -package_key $package_key \ - -locale $locale \ - -charset $mime_charset] - - if { [file exists $alternate_file_path] } { - lappend catalog_file_list $alternate_file_path - } else { - set error_text "lang::catalog::import_from_files - No catalog file found for locale $locale and charset ${mime_charset}. Attempted both path $file_path and $alternate_file_path" - if { ![string equal $charset $mime_charset] } { - append error_text " (defaulted to $charset as $mime_charset is unsupported)" - } - ns_log Error $error_text - } - } else { - # The file doesn't exist and we have no alternate charset to attempt - fail - ns_log Error "lang::catalog::import_from_files - No catalog file found for locale $locale and charset ${mime_charset}. Attempted path $file_path" - } + ns_log Error "Catalog file $file_path not found. Failed to import messages for package $package_key and locale $locale" } } - # Issue a warning and exit if there are no catalog files - if { [empty_string_p $catalog_file_list] } { - ns_log Warning "lang::catalog::import_from_files - No catalog files found for package $package_key" - return - } - - # Loop over each catalog file - foreach file_path $catalog_file_list { - - # First make sure this is really a message catalog file and not some other xml file in the catalog - # directory like a file with saved messages from an upgrade - if { ![apm_is_catalog_file $file_path] } { - # If this doesn't seem to be a file with saved messages from a backup - issue a warning as - # it might be a catalog file on invalid format (for example because of misspelling) - if { ![is_upgrade_backup_file $file_path] } { - ns_log Warning "lang::catalog::import_from_files File $file_path is not on valid message catalog format and is therefore ignored" - } - - continue - } - - # Use a catch so that parse failure of one file doesn't cause the import of all files to fail - if { [catch {import_messages_from_file $file_path} errMsg] } { - global errorInfo - - ns_log Error "lang::catalog::import_from_files - The import of file $file_path failed, error message is:\n\n${errMsg}\n\nstack trace:\n\n$errorInfo\n\n" - } - } + return $catalog_files } -ad_proc -public -deprecated -warn lang::catalog::import_from_tcl_files { - {package_key "acs-lang"} -} { - Import catalog files by evaluating tcl files containing - invocations of the _mr register procedure. Catalog files - should be stored in the /packages/package_key/catalog directory - and have the ending .cat (i.e. /package/dotlrn/catalog/dotlrn.en_US.iso-8859-1.cat). - This procedure is depreceted and has been superseeded by the procedure - lang::catalog::import_from_files that imports catalog files on xml syntax. +################## +# +# Inactive and unmaintained procs +# +################## - @author Jeff Davis - @author Peter Marklund (peter@collaboraid.biz) - @return Number of files loaded - - @see lang::catalog::import_from_files - -} { - set glob_pattern [file join [acs_package_root_dir $package_key] catalog *.cat] - ns_log Notice "Starting load of the message catalogs $glob_pattern" - - global __lang_catalog_load_package_key - set __lang_catalog_load_package_key $package_key - - set files [glob -nocomplain $glob_pattern] - - if {[empty_string_p $files]} { - ns_log Warning "no files found in message catalog directory" - } else { - foreach msg_file $files { - - set src [read_file $msg_file] - - if {[catch {eval $src} errMsg]} { - ns_log Warning "Failed loading message catalog $msg_file:\n$errMsg" - } - } - } - - ns_log Notice "Finished load of the message catalog" - - unset __lang_catalog_load_package_key - - return $files -} - -ad_proc -public lang::catalog::import_from_all_files_and_cache {} { - Loops over all installed and enabled packages that don't already have messages in the database - and imports messages from the catalog files of each such package. When this process is done - the message cache is reloaded. The proc checks if it has been executed before and will - only execute once. - - @author Peter Marklund (peter@collaboraid.biz) -} { - # Only execute this proc once - if { ![nsv_exists lang_catalog_import_from_all_files_and_cache executed_p] } { - nsv_set lang_catalog_import_from_all_files_and_cache executed_p 1 - - db_foreach all_enabled_not_loaded_packages {} { - if { [file isdirectory [file join [acs_package_root_dir $package_key] catalog]] } { - lang::catalog::import_from_files $package_key - } - } - - lang::message::cache - } -} - -ad_proc -private lang::catalog::translate {} { +ad_proc -private lang::catalog::translate {} { Translates all untranslated strings in a message catalog from English into Spanish, French and German - using Babelfish. Quick way to get a multilingual site up and + using Babelfish. NOTE: this proc is unmaintained. + Quick way to get a multilingual site up and running if you can live with the quality of the translations.
Not a good idea to run this procedure if you have @@ -867,40 +778,3 @@ } } } - - -##### -# -# Backwards compatibility procs -# -##### - -ad_proc -deprecated -warn lang_catalog_load_all {} { - @see lang::catalog::import_from_all_files -} { - return [lang::catalog::import_from_all_files] -} - -ad_proc -deprecated -warn lang_catalog_load { - {package_key "acs-lang"} -} { - @see lang::catalog::import_from_files -} { - return [lang::catalog::import_from_files $package_key] -} - -ad_proc -deprecated -warn lang_translate_message_catalog {} { - Translates all untranslated strings in a message catalog - from English into Spanish, French and German - using Babelfish. Quick way to get a multilingual site up and - running if you can live with the quality of the translations. -
- Not a good idea to run this procedure if you have - a large message catalog. Use for testing purposes only. - - @author John Lowry (lowry@arsdigita.com) - - @see lang::catalog::translate -} { - return [lang::catalog::translate] -}