Index: openacs-4/packages/acs-admin/tcl/apm-admin-procs.tcl =================================================================== RCS file: /usr/local/cvsroot/openacs-4/packages/acs-admin/tcl/apm-admin-procs.tcl,v diff -u -r1.32.2.10 -r1.32.2.11 --- openacs-4/packages/acs-admin/tcl/apm-admin-procs.tcl 21 Apr 2023 12:27:29 -0000 1.32.2.10 +++ openacs-4/packages/acs-admin/tcl/apm-admin-procs.tcl 18 Oct 2023 08:59:57 -0000 1.32.2.11 @@ -671,6 +671,857 @@ return 0 } +ad_proc -private apm_git_repo_branches { + -path:required +} { + Extracts the available branches from an OpenACS Git repo. This is + assumes the specific Git setup for our repo, hence it is meant for + internal use only. + + @return list of branch names +} { + set cd_helper [file join $::acs::rootdir bin cd-helper] + set git_command git + + set output [exec $cd_helper $path $git_command branch -r] + + set branches [list] + foreach line [split $output \n] { + if {[regexp {^\s+origin/(oacs-\d+-\d+).*$} $line _ branch]} { + lappend branches $branch + } + } + + return $branches +} + +ad_proc -private apm_git_build_repository { + {-debug:boolean 0} + {-force_fresh:boolean false} + {-channels *} +} { + Rebuild the repository on the local machine. + Only useful for the openacs.org site. + + Adapted from the CVS implementation, which came from Lars' + build-repository.tcl page. + + @param debug Set to 1 to test with only a small subset of packages + and branches instead of all of them. + @param force_fresh Force a frech clone of the Git repos. + @param channels A string match style pattern. Generate apm files + for the matching channels only +} { + + #---------------------------------------------------------------------- + # Configuration Settings + #---------------------------------------------------------------------- + + set sep [file separator] + + set cd_helper [file join $::acs::rootdir bin cd-helper] + + set git_command git + set git_url https://github.com/openacs + + set work_dir [file join $::acs::rootdir repository-builder]${sep} + + set repository_dir [file join $::acs::rootdir www repository]${sep} + set repository_url https://openacs.org/repository/ + + set exclude_package_list {} + + set channel_index_template [template::themed_template /packages/acs-admin/www/apm/repository-channel-index] + set index_template [template::themed_template /packages/acs-admin/www/apm/repository-index] + + + # + # Make sure workdir exists. Clear it before we start if requested. + # + if {$force_fresh_p} { + file delete -force -- $work_dir + } + + file mkdir $work_dir + + #---------------------------------------------------------------------- + # Prepare output + #---------------------------------------------------------------------- + + ns_log Debug "Repository: Building Package Repository" + + #---------------------------------------------------------------------- + # Find available channels + #---------------------------------------------------------------------- + + # + # We first checkout the core repository. This will be the skeleton + # of our mirror. + # + set core_repo_dir ${work_dir}openacs-core + if {[file isdirectory $core_repo_dir]} { + # + # Folder exists. We fetch from the repo to see if new branches + # exist. + # + ns_log notice "Fetching new branches for core repository" + exec -ignorestderr -- $cd_helper $core_repo_dir $git_command fetch origin + } else { + # + # Folder does not exist. We check out the repo from scratch. + # + ns_log notice "Cloning core repository at $git_url/openacs-core.git" + exec -ignorestderr -- $cd_helper $work_dir $git_command clone $git_url/openacs-core.git + } + + # + # The core repo is considered the source of truth concerning + # release branches. We extract them from here and we will look for + # them in the non-core repos. + # + set core_channels [list] + foreach branch [apm_git_repo_branches -path $core_repo_dir] { + regsub {^oacs-} $branch {} channel + if {[string match $channels $channel]} { + lappend core_channels $channel $branch + } + } + lappend core_channels main main + + if {$debug_p} { + # + # When debugging, only pick the last branch. + # + set core_channels [lrange $core_channels end-1 end] + } + + # + # The core packages are those included in the openacs-core + # repository. + # + set core_packages_dir ${core_repo_dir}${sep}packages + + set core_packages [list] + foreach package_folder [glob \ + -types d \ + -directory $core_packages_dir *] { + lappend core_packages [file tail $package_folder] + } + ns_log notice "Core packages:" $core_packages + + set non_core_packages_dir ${work_dir}openacs-non-core + file mkdir $non_core_packages_dir + + # + # This is the list of all packages that are not included in the + # openacs-core repository. We currently maintain this list as + # hardcoded here. One improvement would be to fetch it from the + # Git host directly, either via scraping or via API. + # + # As long as this does not change, everytime a new package is + # added to the Git mirror, one should also add the corresponding + # package key to this list. + # + set non_core_packages { + accounts-desk + accounts-finance + accounts-ledger + accounts-payables + accounts-payroll + accounts-receivables + acs-datetime + acs-events + acs-interface + acs-ldap-authentication + acs-mail + acs-notification + acs-object-management + acs-object-managment + acs-outdated + acs-person + address-book + adserver + ae-portlet + ajax-filestorage-ui + ajax-photoalbum-ui + ajaxhelper + ams + anon-eval + application-track + application-track-portlet + assessment + assessment-portlet + attachments + attendance + auth-cas + auth-http + auth-ldap + auth-pam + auth-server + authorize-gateway + bboard-portlet + beehive + beehive-portlet + bm-portlet + bookmarks + bookshelf + boomerang + bootstrap-icons + bug-tracker + bulk-mail + caldav + calendar + calendar-includelet + calendar-portlet + captcha + cards + cards-portlet + categories + chat + chat-includelet + chat-portlet + clickthrough + clipboard + cms + cms-news-demo + connections + contacts + contacts-lite + contacts-portlet + content-includelet + content-portlet + cookie-consent + courses + cronjob + curriculum + curriculum-central + curriculum-portlet + curriculum-tracker + customer-service + datamanager + datamanager-portlet + dbm + diagram + directory + docker-s6 + dotfolio + dotfolio-ui + dotkul + dotkul-admin + dotlrn + dotlrn-admin + dotlrn-ae + dotlrn-application-track + dotlrn-assessment + dotlrn-attendance + dotlrn-bboard + dotlrn-beehive + dotlrn-bm + dotlrn-calendar + dotlrn-cards + dotlrn-catalog + dotlrn-chat + dotlrn-contacts + dotlrn-content + dotlrn-curriculum + dotlrn-datamanager + dotlrn-dotlrn + dotlrn-ecommerce + dotlrn-edit-this-page + dotlrn-eduwiki + dotlrn-evaluation + dotlrn-expense-tracking + dotlrn-faq + dotlrn-forums + dotlrn-fs + dotlrn-glossar + dotlrn-homework + dotlrn-imsld + dotlrn-invoices + dotlrn-jabber + dotlrn-lamsint + dotlrn-latest + dotlrn-learning-content + dotlrn-lorsm + dotlrn-messages + dotlrn-mmplayer + dotlrn-news + dotlrn-news-aggregator + dotlrn-photo-album + dotlrn-portlet + dotlrn-project-manager + dotlrn-quota + dotlrn-random-photo + dotlrn-recruiting + dotlrn-research + dotlrn-static + dotlrn-survey + dotlrn-syllabus + dotlrn-tasks + dotlrn-user-tracking + dotlrn-weblogger + dotlrn-wikipedia + dotlrn-wps + dotlrn-xowiki + dotlrndoc + download + dynamic-types + ec-serial-numbers + ecommerce + edit-this-page + edit-this-page-portlet + eduwiki + eduwiki-portlet + email-handler + evaluation + evaluation-portlet + expense-tracking + expenses + ezic-gateway + fa-icons + fabrik + facebook-api + faq + faq-portlet + feed-parser + file-manager + file-storage + file-storage-includelet + forums + forums-includelet + forums-portlet + fs-portlet + gatekeeper + general-comments + glossar + glossar-portlet + glossary + highcharts + image-magick + ims-ent + imsld + imsld-portlet + inventory-control + invoices + invoices-portlet + jabber + jabber-portlet + lab-report + lab-report-central + lams-conf + lamsint + lamsint-portlet + lars-blogger + latest + latest-portlet + layout-managed-subsite + layout-manager + learning-content + learning-content-portlet + logger + lors + lors-central + lorsm + lorsm-includelet + lorsm-portlet + mail-tracking + messages + messages-portlet + mmplayer + mmplayer-portlet + monitoring + new-portal + news + news-aggregator + news-aggregator-portlet + news-includelet + news-portlet + notes + oacs-dav + oct-election + online-catalog + openacs-bootstrap3-theme + openacs-bootstrap5-theme + openfts-driver + organizations + package-builder + page + pages + payflowpro + payment-gateway + photo-album + photo-album-portlet + places + planner + poll + postal-address + postcard + press + proctoring-support + profile-provider + project-manager + project-manager-portlet + quota + quota-portlet + random-photo-portlet + ratings + recruiting + recruiting-portlet + redirect + ref-currency + ref-gifi + ref-itu + ref-unspec + ref-us-counties + ref-us-states + ref-us-zipcodes + related-items + research-portlet + richtext-ckeditor4 + richtext-ckeditor5 + richtext-tinymce + richtext-xinha + robot-detection + rss-support + rules + s5 + sample-gateway + schema-browser + scholarship-fund + scorm-core + scorm-importer + scorm-player + scorm-simple-lms + shipping-gateway + shipping-tracking + simple-survey + simulation + site-wide-search + skin + sloan-bboard + soap-db + soap-gateway + spam + spreadsheet + static-pages + static-portlet + survey + survey-builder-ui + survey-library + survey-portlet + survey-reports + t-account + tasks + tasks-portlet + telecom-number + theme-selva + theme-zen + timezones + trackback + tracker + tsoap + user-preferences + user-profile + user-tracking + user-tracking-portlet + value-based-shipping + version-control + views + weblogger-portlet + webmail + webmail-system + wiki + wikipedia + wikipedia-portlet + workflow + wp-slim + wps-portlet + xcms-ui + xml-rpc + xolp + xooauth + xotcl-core + xotcl-request-monitor + xowf + xowf-monaco-plugin + xowiki + xowiki-includelet + xowiki-portlet + } + + if {$debug_p} { + # + # When debugging, pick only a subset of all packages. + # + set non_core_packages [lrange $non_core_packages 0 10] + } + + foreach package_key $non_core_packages { + set package_dir ${non_core_packages_dir}${sep}${package_key} + if {[file isdirectory $package_dir]} { + # + # Folder exists. We fetch from the repo to see if new branches + # exist. + # + ns_log notice "Fetching new branches for non-core repository '$package_key'" + exec -ignorestderr -- $cd_helper $package_dir $git_command fetch origin + } else { + # + # Folder does not exist. Clone the repo from + # scratch. Tolerate errors here, as some legacy packages + # require authentication and would fail. + # + try { + exec -ignorestderr -- $cd_helper $non_core_packages_dir $git_command clone ${git_url}/${package_key}.git + } on error {errmsg} { + ns_log warning "Could not clone '$package_key' from ${git_url}/${package_key}.git:" $errmsg + } + } + } + + + #---------------------------------------------------------------------- + # Read all package .info files, building manifest file + #---------------------------------------------------------------------- + + set update_pretty_date [lc_time_fmt [clock format [clock seconds] -format "%Y-%m-%d %T"] %c] + + foreach {channel branch} $core_channels { + ns_log Notice "Repository: Channel $channel using branch $branch" + + # + # Checkout the channel branch on the core repository. + # + ns_log Notice "Checking out core-repository" + exec -ignorestderr -- $cd_helper $core_repo_dir $git_command checkout $branch + # + # Make sure the repo is up to date. + # + ns_log Notice "Updating core-repository" + exec -ignorestderr -- $cd_helper $core_repo_dir $git_command pull + + # + # Try to check out the channel from the non-core packages. + # + set branch_packages [list] + foreach package_key $non_core_packages { + set package_dir ${non_core_packages_dir}${sep}${package_key} + if {![file isdirectory $package_dir]} { + ns_log notice "Package '$package_key' was not cloned in '$package_dir', skipping." + continue + } + + # + # Not all packages will have a release branch. Skip the + # package when the branch is not found. + # + if {$branch in [apm_git_repo_branches -path $package_dir]} { + try { + ns_log Notice "Checking out '$package_key'" + exec -ignorestderr -- $cd_helper $package_dir $git_command checkout $branch + } on error {errmsg} { + # + # Checking out a branch that was already checked + # out will complain. As we know the branch exists + # for this repo, we are pretty confident this + # error can be ignored. + # + ns_log notice "Checking out existing branch '$branch' for package '$package_key' complained:" $errmsg + } + # + # Make sure repo is up to date. + # + ns_log Notice "Updating '$package_key'" + exec -ignorestderr -- $cd_helper $package_dir $git_command pull + + lappend branch_packages $package_key + } + } + + # + # Now collect the info files for all core and non-core + # packages belonging to this branch. + # + set info_files [list] + foreach package_key $core_packages { + if {[catch { + set info_file [apm_package_info_file_path -path $core_packages_dir $package_key] + } errmsg]} { + ns_log warning "Cannot find an .info file on '$branch' for core package '$package_key':" $errmsg + continue + } + + lappend info_files $info_file + } + foreach package_key $branch_packages { + if {[catch { + set info_file [apm_package_info_file_path -path $non_core_packages_dir $package_key] + } errmsg]} { + ns_log warning "Cannot find an .info file on '$branch' for non.core package '$package_key':" $errmsg + continue + } + + lappend info_files $info_file + } + + # Prepare channel directory + set channel_dir "${work_dir}repository${sep}${channel}${sep}" + file mkdir $channel_dir + + set manifest "\n" + + template::multirow create packages \ + package_path package_key version pretty_name \ + package_type summary description \ + release_date vendor_url vendor \ + maturity maturity_text \ + license license_url + + set packages [list] + + foreach spec_file [lsort $info_files] { + + set package_path [file join {*}[lrange [file split $spec_file] 0 end-1]] + set package_key [lindex [file split $spec_file] end-1] + + if { $package_key in $exclude_package_list } { + ns_log Debug "Repository: Package $package_key is on list of packages to exclude - skipping" + continue + } + + if { [array exists pkg_info] } { + array unset pkg_info + } + if { [info exists pkg_info] } { + unset pkg_info + } + + ad_try { + array set pkg_info [apm_read_package_info_file $spec_file] + + if { $pkg_info(package.key) in $packages } { + ns_log Debug "Repository: Skipping package $package_key, because we already have another version of it" + } else { + lappend packages $pkg_info(package.key) + + append manifest \ + " " \n \ + " [ns_quotehtml $pkg_info(package.key)]\n" \ + " [ns_quotehtml $pkg_info(name)]\n" \ + " [ns_quotehtml $pkg_info(package-name)]\n" \ + " [ns_quotehtml $pkg_info(package.type)]\n" \ + " [ns_quotehtml $pkg_info(summary)]\n" \ + " " \ + [ns_quotehtml $pkg_info(description)] "\n" \ + " [ns_quotehtml $pkg_info(release-date)]\n" \ + " " \ + [ns_quotehtml $pkg_info(vendor)] "\n" \ + " " \ + [ns_quotehtml $pkg_info(license)] "\n" \ + " $pkg_info(maturity)\n" + + foreach e $pkg_info(install) { + append manifest " \n" + } + + template::multirow append packages \ + $package_path $package_key $pkg_info(name) $pkg_info(package-name) \ + $pkg_info(package.type) $pkg_info(summary) $pkg_info(description) \ + $pkg_info(release-date) $pkg_info(vendor.url) $pkg_info(vendor) \ + $pkg_info(maturity) $pkg_info(maturity_text) \ + $pkg_info(license) $pkg_info(license.url) + + set apm_file "${channel_dir}${pkg_info(package.key)}-${pkg_info(name)}.apm" + ns_log Notice "Repository: Building package $package_key for channel $channel" + + set files [apm_get_package_files \ + -all \ + -include_data_model_files \ + -all_db_types \ + -package_key $pkg_info(package.key) \ + -package_path $package_path] + + if { [llength $files] == 0 } { + ns_log Notice "Repository: No files in package" + } else { + ns_log Notice "Repository: [llength $files] files in package $pkg_info(package.key) ($channel)" + set cmd [list exec [apm_tar_cmd] cf - 2>/dev/null] + + # The path to the 'packages' directory in the checkout + set packages_root_path [file join {*}[lrange [file split $spec_file] 0 end-2]] + + set fp [ad_opentmpfile tmp_filename] + foreach file $files { + puts $fp $package_key/$file + } + close $fp + + lappend cmd -C $packages_root_path --files-from $tmp_filename + + lappend cmd "|" [apm_gzip_cmd] -c ">" $apm_file + ns_log Notice "Executing: exec $cd_helper $packages_root_path $cmd" + if {[catch "exec $cd_helper $packages_root_path $cmd" errmsg]} { + ns_log Error "Error during tar in repository creation for\ + file ${channel_dir}$pkg_info(package.key)-$pkg_info(name).apm:\ + \n$errmsg\n$::errorCode,$::errorInfo" + } + file delete -- $tmp_filename + } + + set apm_url "${repository_url}$channel/$pkg_info(package.key)-$pkg_info(name).apm" + + append manifest " $apm_url\n" + foreach elm $pkg_info(provides) { + append manifest " \n" + } + + foreach elm $pkg_info(requires) { + append manifest " \n" + } + append manifest " \n" + } + } on error {errorMsg} { + ns_log Notice "Repository: Error on spec_file $spec_file: $errorMsg\n$::errorInfo\n" + } + } + + append manifest "\n" + + ns_log Notice "Repository: Writing $channel manifest to ${channel_dir}manifest.xml" + set fw [open "${channel_dir}manifest.xml" w] + puts $fw $manifest + close $fw + + ns_log Notice "Repository: Writing $channel index page to ${channel_dir}index.adp" + set fw [open "${channel_dir}index.adp" w] + set packages [lsort $packages] + puts $fw "\nOpenACS $channel Compatible Packages\n\n" + puts $fw "

OpenACS $channel (Git branch $branch)

+

Packages can be installed with the OpenACS Automated Installer on + your OpenACS site at /acs-admin/install. Only packages + potentially compatible with your OpenACS kernel will be shown.

+ " + set category_title(core) "Core Packages" + set package_keys(core) $core_packages + + set category_title(common-app) "Common Applications" + set package_keys(common-app) { + xowiki + xotcl-request-monitor + file-storage + acs-developer-support + forums + calendar + news + faq + } + + set category_title(extra) "Extra Packages and Libraries" + set package_keys(extra) "" + foreach p $packages { + if {$p ni $package_keys(core) && $p ni $package_keys(common-app)} { + lappend package_keys(extra) $p + } + } + + foreach category {core common-app extra} { + + template::multirow create pkgs \ + package_path package_key version pretty_name \ + package_type summary description \ + release_date vendor_url vendor \ + maturity maturity_text \ + license license_url + + template::multirow foreach packages { + if {$package_key in $package_keys($category)} { + template::multirow append pkgs \ + $package_path $package_key $version $pretty_name \ + $package_type $summary $description \ + $release_date $vendor_url $vendor \ + $maturity $maturity_text \ + $license $license_url + } + } + + puts $fw "\n

$category_title($category)

\n" + + puts $fw [template::adp_include $channel_index_template \ + [list channel $channel &pkgs pkgs update_pretty_date $update_pretty_date]] + + } + close $fw + + ns_log Notice "Repository: Channel $channel complete." + + } + + ns_log Notice "Repository: Finishing Repository" + + foreach channel [dict keys $core_channels] { + if {[regexp {^([1-9][0-9]*)-([0-9]+)$} $channel . major minor]} { + # + # *-compat channels: The "patchlevel" of these channels is + # the highest possible value, higher than the released + # -final channels. + # + set tag_order([format %.3d $major]-[format %.3d $minor]-999) $channel + set tag_label($channel) "OpenACS $major.$minor" + } elseif {[regexp {^([1-9][0-9]*)-([0-9]+)-([0-9]+)$} $channel . major minor patch]} { + # + # *-final channels: a concrete patchlevel is provided. + # + set tag_order([format %.3d $major]-[format %.3d $minor]-[format %.3d $patch]) $channel + set tag_label($channel) "OpenACS $major.$minor.$patch" + } else { + set tag_order(999-999-999) $channel + set tag_label($channel) "OpenACS $channel" + } + } + + + # Write the index page + ns_log Notice "Repository: Writing repository index page to ${work_dir}repository/index.adp" + template::multirow create channels name tag label + foreach key [lsort -decreasing [array names tag_order]] { + set channel $tag_order($key) + template::multirow append channels $channel [dict get $core_channels $channel] $tag_label($channel) + } + set fw [open "${work_dir}repository/index.adp" w] + puts $fw "\nOpenACS Package Repository\n\n" + puts $fw [template::adp_include -- $index_template \ + [list &channels channels update_pretty_date $update_pretty_date]] + close $fw + + # Add a redirector for outdated releases + set fw [open "${work_dir}repository/index.vuh" w] + puts $fw "ns_returnredirect /repository/" + close $fw + + # Without the trailing slash + set work_repository_dirname "${work_dir}repository" + set repository_dirname [string range $repository_dir 0 end-1] + set repository_bak "[string range $repository_dir 0 end-1]_bak" + + ns_log Notice "Repository: Moving work repository $work_repository_dirname to live repository dir at $repository_dir\n" + + if { [file exists $repository_bak] } { + file delete -force -- $repository_bak + } + if { [file exists $repository_dirname] } { + file rename -- $repository_dirname $repository_bak + } + file rename -- $work_repository_dirname $repository_dirname + + ns_log Debug "Repository: DONE" + + return 0 +} + # # Local variables: # mode: tcl