Index: openacs-4/packages/acs-admin/www/apm/packages-install-3.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-admin/www/apm/packages-install-3.tcl,v diff -u -N -r1.7 -r1.8 --- openacs-4/packages/acs-admin/www/apm/packages-install-3.tcl 17 May 2003 09:36:29 -0000 1.7 +++ openacs-4/packages/acs-admin/www/apm/packages-install-3.tcl 16 Jul 2003 00:15:31 -0000 1.8 @@ -46,16 +46,19 @@ # Find out which script is appropriate to be run. set data_model_in_package 0 set table_rows "" - set data_model_files [apm_data_model_scripts_find \ - -upgrade_from_version_name $initial_version_name \ - -upgrade_to_version_name $final_version_name \ - $package_key] + set data_model_files [concat \ + [apm_data_model_scripts_find \ + -upgrade_from_version_name $initial_version_name \ + -upgrade_to_version_name $final_version_name \ + $package_key] \ + [apm_ctl_files_find $package_key]] + set sql_file_list [concat $sql_file_list $data_model_files] + if {![empty_string_p $data_model_files]} { foreach file $data_model_files { set path [lindex $file 0] set file_type [lindex $file 1] - set db_type [lindex $file 2] append table_rows " $path @@ -75,8 +78,8 @@ - + $table_rows
LoadFile Type File NameFile Type
Index: openacs-4/packages/acs-bootstrap-installer/tcl/30-apm-load-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-bootstrap-installer/tcl/30-apm-load-procs.tcl,v diff -u -N -r1.19 -r1.20 --- openacs-4/packages/acs-bootstrap-installer/tcl/30-apm-load-procs.tcl 12 Jul 2003 01:10:35 -0000 1.19 +++ openacs-4/packages/acs-bootstrap-installer/tcl/30-apm-load-procs.tcl 16 Jul 2003 00:15:31 -0000 1.20 @@ -47,6 +47,7 @@
  1. Files with extension .sql are considered data-model files,
  2. Files with extension .csv are considered comma-separated values files. +
  3. Files with extension .ctl are considered sql data loader control files. or if any path contains the substring upgrade, data-model upgrade files.
  4. Files with extension .sqlj are considered sqlj_code files. @@ -112,6 +113,8 @@ } } elseif { [string equal $extension ".csv"] } { set type "csv_data" + } elseif { [string equal $extension ".ctl"] } { + set type "ctl_file" } elseif { [string equal $extension ".sqlj"] } { set type "sqlj_code" } elseif { [string equal $extension ".info"] } { Index: openacs-4/packages/acs-kernel/acs-kernel.info =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-kernel/acs-kernel.info,v diff -u -N -r1.26 -r1.27 --- openacs-4/packages/acs-kernel/acs-kernel.info 19 May 2003 01:15:27 -0000 1.26 +++ openacs-4/packages/acs-kernel/acs-kernel.info 16 Jul 2003 00:15:31 -0000 1.27 @@ -7,7 +7,7 @@ t t - + oracle postgresql @@ -17,7 +17,7 @@ 2003-02-18 OpenACS - + Index: openacs-4/packages/acs-kernel/sql/oracle/apm-create.sql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-kernel/sql/oracle/apm-create.sql,v diff -u -N -r1.18 -r1.19 --- openacs-4/packages/acs-kernel/sql/oracle/apm-create.sql 15 Jul 2003 00:38:42 -0000 1.18 +++ openacs-4/packages/acs-kernel/sql/oracle/apm-create.sql 16 Jul 2003 00:15:31 -0000 1.19 @@ -564,6 +564,7 @@ insert into apm_package_file_types(file_type_key, pretty_name) values('content_page', 'Content page'); insert into apm_package_file_types(file_type_key, pretty_name) values('package_spec', 'Package specification'); insert into apm_package_file_types(file_type_key, pretty_name) values('csv_data', 'CSV Data'); + insert into apm_package_file_types(file_type_key, pretty_name) values('ctl_file', 'SQL data loader control'); insert into apm_package_file_types(file_type_key, pretty_name) values('data_model', 'Data model'); insert into apm_package_file_types(file_type_key, pretty_name) values('data_model_create', 'Data model installation'); insert into apm_package_file_types(file_type_key, pretty_name) values('data_model_drop', 'Data model deinstallation'); Index: openacs-4/packages/acs-kernel/sql/oracle/upgrade/upgrade-4.7.2d-5.0d.sql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-kernel/sql/oracle/upgrade/upgrade-4.7.2d-5.0d.sql,v diff -u -N -r1.1 -r1.2 --- openacs-4/packages/acs-kernel/sql/oracle/upgrade/upgrade-4.7.2d-5.0d.sql 15 Jul 2003 00:38:42 -0000 1.1 +++ openacs-4/packages/acs-kernel/sql/oracle/upgrade/upgrade-4.7.2d-5.0d.sql 16 Jul 2003 00:15:31 -0000 1.2 @@ -1 +1,2 @@ insert into apm_package_file_types(file_type_key, pretty_name) values('csv_data', 'CSV Data'); +insert into apm_package_file_types(file_type_key, pretty_name) values('ctl_file', 'SQL data loader control'); Index: openacs-4/packages/acs-kernel/sql/postgresql/apm-create.sql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-kernel/sql/postgresql/apm-create.sql,v diff -u -N -r1.38 -r1.39 --- openacs-4/packages/acs-kernel/sql/postgresql/apm-create.sql 15 Jul 2003 00:38:42 -0000 1.38 +++ openacs-4/packages/acs-kernel/sql/postgresql/apm-create.sql 16 Jul 2003 00:15:32 -0000 1.39 @@ -739,7 +739,8 @@ insert into apm_package_file_types(file_type_key, pretty_name) values(''tcl_util'', ''Tcl utility script''); insert into apm_package_file_types(file_type_key, pretty_name) values(''content_page'', ''Content page''); insert into apm_package_file_types(file_type_key, pretty_name) values(''package_spec'', ''Package specification''); - insert into apm_package_file_types(file_type_key, pretty_name) values(''csv_data'', ''CSV Data''); + insert into apm_package_file_types(file_type_key, pretty_name) values(''ctl_file'', ''SQL data loader control''); + insert into apm_package_file_types(file_type_key, pretty_name) values(''csv_data'', ''CSV data''); insert into apm_package_file_types(file_type_key, pretty_name) values(''data_model'', ''Data model''); insert into apm_package_file_types(file_type_key, pretty_name) values(''data_model_create'', ''Data model installation''); insert into apm_package_file_types(file_type_key, pretty_name) values(''data_model_drop'', ''Data model deinstallation''); Index: openacs-4/packages/acs-kernel/sql/postgresql/upgrade/upgrade-4.7.2d-5.0d.sql =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-kernel/sql/postgresql/upgrade/upgrade-4.7.2d-5.0d.sql,v diff -u -N -r1.2 -r1.3 --- openacs-4/packages/acs-kernel/sql/postgresql/upgrade/upgrade-4.7.2d-5.0d.sql 15 Jul 2003 00:39:37 -0000 1.2 +++ openacs-4/packages/acs-kernel/sql/postgresql/upgrade/upgrade-4.7.2d-5.0d.sql 16 Jul 2003 00:15:32 -0000 1.3 @@ -1 +1,2 @@ insert into apm_package_file_types(file_type_key, pretty_name) values('csv_data', 'CSV Data'); +insert into apm_package_file_types(file_type_key, pretty_name) values('ctl_file', 'SQL data loader control'); Index: openacs-4/packages/acs-tcl/tcl/00-database-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-tcl/tcl/00-database-procs.tcl,v diff -u -N -r1.30 -r1.31 --- openacs-4/packages/acs-tcl/tcl/00-database-procs.tcl 7 Jul 2003 12:00:01 -0000 1.30 +++ openacs-4/packages/acs-tcl/tcl/00-database-procs.tcl 16 Jul 2003 00:15:32 -0000 1.31 @@ -2008,7 +2008,6 @@ } } - ad_proc db_get_pgbin {{ -dbn "" }} { PostgreSQL only. @@ -2093,7 +2092,6 @@ return [string range $datasource 0 [expr $first_colon_pos - 1]] } - ad_proc db_source_sql_file {{ -dbn "" -callback apm_ns_write_callback @@ -2209,7 +2207,132 @@ } } +ad_proc db_load_sql_data {{ + -dbn "" + -callback apm_ns_write_callback +} file } { + Loads a CSV formatted file into a table using PostgreSQL's COPY command or + Oracle's SQL*Loader utility. The file name format consists of a sequence + number used to control the order in which tables are loaded, and the table + name with "-" replacing "_". This is a bit of a kludge but greatly speeds + the loading of large amounts of data, such as is done when various "ref-*" + packages are installed. + + @param dbn The database name to use. If empty_string, uses the default database. + @file Filename in the format dd-table-name.csv where 'dd' is a sequence number + used to control the order in which data is loaded, and 'table-name' is + the name of the SQL table to be loaded with '-' replacing '_'. + +} { + + switch [db_driverkey $dbn] { + + oracle { + global env + + set user_pass [db_get_sql_user -dbn $dbn] + set tmpnam [ns_tmpnam] + + set fd [open $file r] + set fd1 [open $tmpnam w] + write $fd1 [subst [read $fd]] + close $fd1 + close $fd + + cd [file dirname $file] + set fd [open "|[file join $env(ORACLE_HOME) bin sqlldr] userid=$user_pass control=$tmpnam" "r"] + + while { [gets $fd line] >= 0 } { + # Don't bother writing out lines which are purely whitespace. + if { ![string is space $line] } { + apm_callback_and_log $callback "[ad_quotehtml $line]\n" + } + } + close $fd + } + + postgresql { + global tcl_platform + + set pguser [db_get_username] + if { ![string equal $pguser ""] } { + set pguser "-U $pguser" + } + + set pgport [db_get_port] + if { ![string equal $pgport ""] } { + set pgport "-p $pgport" + } + + set pgpass [db_get_password] + if { ![string equal $pgpass ""] } { + set pgpass "<<$pgpass" + } + + if { [string equal [db_get_dbhost] "localhost"] || [string equal [db_get_dbhost] ""] } { + set pghost "" + } else { + set pghost "-h [db_get_dbhost]" + } + + set fd [open $file r] + set copy_command [subst [read $fd]] + close $fd + + if { $tcl_platform(platform) == "windows" } { + set fp [open "|[file join [db_get_pgbin] psql] -c \"$copy_command\" -h [ns_info hostname] $pgport $pguser [db_get_database]" "r"] + } else { + set fp [open "|[file join [db_get_pgbin] psql] -c \"$copy_command\" $pghost $pgport $pguser [db_get_database] $pgpass" "r"] + } + + while { [gets $fp line] >= 0 } { + # Don't bother writing out lines which are purely whitespace. + if { ![string is space $line] } { + apm_callback_and_log $callback "[ad_quotehtml $line]\n" + } + } + + # PSQL dumps errors and notice information on stderr, and has no option to turn + # this off. So we have to chug through the "error" lines looking for those that + # really signal an error. + + set errno [ catch { + close $fp + } error] + + if { $errno == 2 } { + return $error + } + + # Just filter out the "NOTICE" lines, so we get the stack dump along with real + # ERRORs. This could be done with a couple of opaque-looking regexps... + + set error_found 0 + foreach line [split $error "\n"] { + if { [string first NOTICE $line] == -1 } { + append error_lines "$line\n" + set error_found [expr { $error_found || [string first ERROR $line] != -1 || \ + [string first FATAL $line] != -1 } ] + } + } + + if { $error_found } { + global errorCode + return -code error -errorinfo $error_lines -errorcode $errorCode $error_lines + } + + } + + nsodbc { + error "db_load_sql_data is not supported for this database." + } + default { + error "db_load_sql_data is not supported for this database." + } + } +} + ad_proc db_source_sqlj_file {{ -dbn "" -callback apm_ns_write_callback Index: openacs-4/packages/acs-tcl/tcl/apm-install-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-tcl/tcl/apm-install-procs.tcl,v diff -u -N -r1.41 -r1.42 --- openacs-4/packages/acs-tcl/tcl/apm-install-procs.tcl 18 Jun 2003 13:11:48 -0000 1.41 +++ openacs-4/packages/acs-tcl/tcl/apm-install-procs.tcl 16 Jul 2003 00:15:32 -0000 1.42 @@ -763,6 +763,21 @@ apm_callback_and_log $callback "\n" } } + + set data_files [apm_ctl_files_find $package_key] + foreach item $data_files { + set file_path [lindex $item 0] + ns_log Debug "APM: Now processing $file_path of type ctl_file" + if { !$ul_p } { + apm_callback_and_log $callback "
      \n" + set ul_p 1 + } + apm_callback_and_log $callback "
    • Loading data file $path/$file_path... +
      "
+        db_load_sql_data -callback $callback $path/$file_path
+	apm_callback_and_log $callback "
      \n" + } + if {$ul_p} { apm_callback_and_log $callback "

    " } @@ -1164,6 +1179,39 @@ } } +ad_proc -private apm_ctl_files_find { + package_key +} { + + Return SQL loader control files for a given package. These will be executed in + alphabetical order so will typically have names with embedded sequence numbers + like "00-foo.tcl" etc. + + @package_key the package to search + @return A list of files and file types of form [list [list "00-foo.ctl" "ctl_file"] ...] + +} { + + set file_list [apm_get_package_files -file_types [list ctl_file] -package_key $package_key] + set files [list] + + foreach path $file_list { + set file_type [apm_guess_file_type $package_key $path] + set file_db_type [apm_guess_db_type $package_key $path] + + apm_log APMDebug "APM: Checking \"$path\" of type \"$file_type\" and db_type \"$file_db_type\"." + + if { [db_compatible_rdbms_p $file_db_type]} { + apm_log APMDebug "APM: Adding $path to the list of data model files." + lappend files [list $path $file_type $package_key ] + } + } + apm_log APMDebug "APM: SQL loader control files for $package_key: $files" + return $files + +} + + ad_proc -private apm_data_model_scripts_find { {-upgrade_from_version_name ""} {-upgrade_to_version_name ""}