#!/usr/bin/perl #------------------------------------------------------------------------------ # Free realtime web server logfile analyzer to show advanced web statistics. # Works from command line or as a CGI. You must use this script as often as # necessary from your scheduler to update your statistics and from command # line or a browser to read report results. # See AWStats documentation (in docs/ directory) for all setup instructions. #------------------------------------------------------------------------------ # $Revision: 1.1.2.1 $ - $Author: davido $ - $Date: 2005/06/28 08:03:12 $ # $Modifications by Sergio Gonzalez, Pablo Muņoz and David Ortega, TID, E-Lane project require 5.005; #$|=1; #use warnings; # Must be used in test mode only. This reduce a little process speed #use diagnostics; # Must be used in test mode only. This reduce a lot of process speed use strict;no strict "refs"; use Time::Local; # use Time::Local 'timelocal_nocheck' is faster but not supported by all Time::Local modules use Socket; #------------------------------------------------------------------------------ # Defines #------------------------------------------------------------------------------ use vars qw/ $REVISION $VERSION /; $REVISION='$Revision: 1.1.2.1 $'; $REVISION =~ /\s(.*)\s/; $REVISION=$1; $VERSION="6.2 (build $REVISION)"; # ----- Constants ----- use vars qw/ $DEBUGFORCED $NBOFLINESFORBENCHMARK $FRAMEWIDTH $NBOFLASTUPDATELOOKUPTOSAVE $LIMITFLUSH $NEWDAYVISITTIMEOUT $VISITTIMEOUT $NOTSORTEDRECORDTOLERANCE $WIDTHCOLICON $TOOLTIPON $lastyearbeforeupdate /; $DEBUGFORCED=0; # Force debug level to log lesser level into debug.log file (Keep this value to 0) $NBOFLINESFORBENCHMARK=8192; # Benchmark info are printing every NBOFLINESFORBENCHMARK lines (Must be a power of 2) $FRAMEWIDTH=240; # Width of left frame when UseFramesWhenCGI is on $NBOFLASTUPDATELOOKUPTOSAVE=500; # Nb of records to save in DNS last update cache file $LIMITFLUSH=5000; # Nb of records in data arrays after how we need to flush data on disk $NEWDAYVISITTIMEOUT=764041; # Delay between 01-23:59:59 and 02-00:00:00 $VISITTIMEOUT=10000; # Lapse of time to consider a page load as a new visit. 10000 = 1 hour (Default = 10000) $NOTSORTEDRECORDTOLERANCE=20000; # Lapse of time to accept a record if not in correct order. 20000 = 2 hour (Default = 20000) $WIDTHCOLICON=32; $TOOLTIPON=0; # Tooltips plugin loaded # ----- Running variables ----- use vars qw/ $DIR $PROG $Extension $Debug $ShowSteps $DebugResetDone $DNSLookupAlreadyDone $RunAsCli $UpdateFor $HeaderHTTPSent $HeaderHTMLSent $LastLine $LastLineNumber $LastLineOffset $LastLineChecksum $LastUpdate $lowerval $PluginMode $TotalUnique $TotalVisits $TotalHostsKnown $TotalHostsUnknown $TotalPages $TotalHits $TotalBytes $TotalNotViewedPages $TotalNotViewedHits $TotalNotViewedBytes $TotalEntries $TotalExits $TotalBytesPages $TotalDifferentPages $TotalKeyphrases $TotalKeywords $TotalDifferentKeyphrases $TotalDifferentKeywords $TotalSearchEnginesPages $TotalSearchEnginesHits $TotalRefererPages $TotalRefererHits $TotalDifferentSearchEngines $TotalDifferentReferer $FrameName $Center $FileConfig $FileSuffix $Host $DayRequired $MonthRequired $YearRequired $QueryString $SiteConfig $StaticLinks $PageCode $PageDir $PerlParsingFormat $UserAgent $pos_vh $pos_host $pos_logname $pos_date $pos_tz $pos_method $pos_url $pos_code $pos_size $pos_referer $pos_agent $pos_query $pos_gzipin $pos_gzipout $pos_compratio $pos_timetaken $pos_cluster $pos_emails $pos_emailr $pos_hostr @pos_extra $lastdate $position $counter $date $counterSmall $positionSmall $i $elements $j $element_copy $k $condition /; $DIR=$PROG=$Extension=''; $Debug = $ShowSteps = 0; $DebugResetDone = $DNSLookupAlreadyDone = 0; $RunAsCli = $UpdateFor = $HeaderHTTPSent = $HeaderHTMLSent = 0; $LastLine = $LastLineNumber = $LastLineOffset = $LastLineChecksum = $LastUpdate = $condition = 0; $lowerval = 0; $PluginMode = ''; $TotalUnique = $TotalVisits = $TotalHostsKnown = $TotalHostsUnknown = 0; $TotalPages = $TotalHits = $TotalBytes = 0; $TotalNotViewedPages = $TotalNotViewedHits = $TotalNotViewedBytes = 0; $TotalEntries = $TotalExits = $TotalBytesPages = $TotalDifferentPages = 0; $TotalKeyphrases = $TotalKeywords = $TotalDifferentKeyphrases = $TotalDifferentKeywords = 0; $TotalSearchEnginesPages = $TotalSearchEnginesHits = $TotalRefererPages = $TotalRefererHits = $TotalDifferentSearchEngines = $TotalDifferentReferer = 0; ($FrameName, $Center, $FileConfig, $FileSuffix, $Host, $DayRequired, $MonthRequired, $YearRequired, $QueryString, $SiteConfig, $StaticLinks, $PageCode, $PageDir, $PerlParsingFormat, $UserAgent)= ('','','','','','','','','','','','','','',''); $pos_vh = $pos_host = $pos_logname = $pos_date = $pos_tz = $pos_method = $pos_url = $pos_code = $pos_size = -1; $pos_referer = $pos_agent = $pos_query = $pos_gzipin = $pos_gzipout = $pos_compratio = -1; $pos_cluster = $pos_emails = $pos_emailr = $pos_hostr = -1; @pos_extra=(); # ----- Plugins variable ----- use vars qw/ %PluginsLoaded $PluginDir $AtLeastOneSectionPlugin /; %PluginsLoaded=(); $PluginDir=''; $AtLeastOneSectionPlugin=0; # ----- Time vars ----- use vars qw/ $starttime $nowtime $tomorrowtime $nowweekofmonth $nowweekofyear $nowdaymod $nowsmallyear $nowsec $nowmin $nowhour $nowday $nowmonth $nowyear $nowwday $nowyday $nowns $StartSeconds $StartMicroseconds /; $StartSeconds=$StartMicroseconds=0; # ----- Variables for config file reading ----- use vars qw/ $FoundNotPageList /; $FoundNotPageList=0; # ----- Config file variables ----- use vars qw/ $StaticExt $DNSStaticCacheFile $DNSLastUpdateCacheFile $MiscTrackerUrl $Lang $MaxRowsInHTMLOutput $MaxLengthOfShownURL $MaxLengthOfStoredURL $MaxLengthOfStoredUA %BarPng $BuildReportFormat $BuildHistoryFormat $ExtraTrackedRowsLimit /; $StaticExt='html'; $DNSStaticCacheFile='dnscache.txt'; $DNSLastUpdateCacheFile='dnscachelastupdate.txt'; $MiscTrackerUrl='/js/awstats_misc_tracker.js'; $Lang='auto'; $MaxRowsInHTMLOutput=1000; $MaxLengthOfShownURL=64; $MaxLengthOfStoredURL=256; # Note: Apache LimitRequestLine is default to 8190 $MaxLengthOfStoredUA=256; %BarPng=('vv'=>'vv.png','vu'=>'vu.png','hu'=>'hu.png','vp'=>'vp.png','hp'=>'hp.png', 'he'=>'he.png','hx'=>'hx.png','vh'=>'vh.png','hh'=>'hh.png','vk'=>'vk.png','hk'=>'hk.png'); $BuildReportFormat='html'; $BuildHistoryFormat='text'; $ExtraTrackedRowsLimit=500; use vars qw/ $EnableLockForUpdate $DNSLookup $AllowAccessFromWebToAuthenticatedUsersOnly $BarHeight $BarWidth $CreateDirDataIfNotExists $KeepBackupOfHistoricFiles $NbOfLinesParsed $NbOfLinesDropped $NbOfLinesCorrupted $NbOfOldLines $NbOfNewLines $NbOfLinesShowsteps $NewLinePhase $NbOfLinesForCorruptedLog $PurgeLogFile $ArchiveLogRecords $ShowDropped $ShowCorrupted $ShowUnknownOrigin $ShowLinksToWhoIs $ShowAuthenticatedUsers $ShowFileSizesStats $ShowScreenSizeStats $ShowSMTPErrorsStats $ShowEMailSenders $ShowEMailReceivers $ShowWormsStats $ShowClusterStats $IncludeInternalLinksInOriginSection $AuthenticatedUsersNotCaseSensitive $Expires $UpdateStats $MigrateStats $URLNotCaseSensitive $URLWithQuery $URLReferrerWithQuery $DecodeUA /; ($EnableLockForUpdate, $DNSLookup, $AllowAccessFromWebToAuthenticatedUsersOnly, $BarHeight, $BarWidth, $CreateDirDataIfNotExists, $KeepBackupOfHistoricFiles, $NbOfLinesParsed, $NbOfLinesDropped, $NbOfLinesCorrupted, $NbOfOldLines, $NbOfNewLines, $NbOfLinesShowsteps, $NewLinePhase, $NbOfLinesForCorruptedLog, $PurgeLogFile, $ArchiveLogRecords, $ShowDropped, $ShowCorrupted, $ShowUnknownOrigin, $ShowLinksToWhoIs, $ShowAuthenticatedUsers, $ShowFileSizesStats, $ShowScreenSizeStats, $ShowSMTPErrorsStats, $ShowEMailSenders, $ShowEMailReceivers, $ShowWormsStats, $ShowClusterStats, $IncludeInternalLinksInOriginSection, $AuthenticatedUsersNotCaseSensitive, $Expires, $UpdateStats, $MigrateStats, $URLNotCaseSensitive, $URLWithQuery, $URLReferrerWithQuery, $DecodeUA)= (0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0); use vars qw/ $AllowToUpdateStatsFromBrowser $DetailedReportsOnNewWindows $FirstDayOfWeek $KeyWordsNotSensitive $SaveDatabaseFilesWithPermissionsForEveryone $WarningMessages $DebugMessages $ShowLinksOnUrl $UseFramesWhenCGI $ShowMenu $ShowMonthStats $ShowDaysOfMonthStats $ShowDaysOfWeekStats $ShowHoursStats $ShowDomainsStats $ShowHostsStats $ShowRobotsStats $ShowSessionsStats $ShowPagesStats $ShowFileTypesStats $ShowOSStats $ShowBrowsersStats $ShowOriginStats $ShowKeyphrasesStats $ShowKeywordsStats $ShowMiscStats $ShowHTTPErrorsStats $AddDataArrayMonthStats $AddDataArrayShowDaysOfMonthStats $AddDataArrayShowDaysOfWeekStats $AddDataArrayShowHoursStats /; ($AllowToUpdateStatsFromBrowser, $DetailedReportsOnNewWindows, $FirstDayOfWeek, $KeyWordsNotSensitive, $SaveDatabaseFilesWithPermissionsForEveryone, $WarningMessages, $DebugMessages, $ShowLinksOnUrl, $UseFramesWhenCGI, $ShowMenu, $ShowMonthStats, $ShowDaysOfMonthStats, $ShowDaysOfWeekStats, $ShowHoursStats, $ShowDomainsStats, $ShowHostsStats, $ShowRobotsStats, $ShowSessionsStats, $ShowPagesStats, $ShowFileTypesStats, $ShowOSStats, $ShowBrowsersStats, $ShowOriginStats, $ShowKeyphrasesStats, $ShowKeywordsStats, $ShowMiscStats, $ShowHTTPErrorsStats, $AddDataArrayMonthStats, $AddDataArrayShowDaysOfMonthStats, $AddDataArrayShowDaysOfWeekStats, $AddDataArrayShowHoursStats )= (1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1); use vars qw/ $AllowFullYearView $LevelForRobotsDetection $LevelForWormsDetection $LevelForBrowsersDetection $LevelForOSDetection $LevelForRefererAnalyze $LevelForFileTypesDetection $LevelForSearchEnginesDetection $LevelForKeywordsDetection /; ($AllowFullYearView, $LevelForRobotsDetection, $LevelForWormsDetection, $LevelForBrowsersDetection, $LevelForOSDetection, $LevelForRefererAnalyze, $LevelForFileTypesDetection, $LevelForSearchEnginesDetection, $LevelForKeywordsDetection)= (2,2,0,2,2,2,2,2,2); use vars qw/ $DirLock $DirCgi $DirConfig $DirData $DirIcons $DirLang $AWScript $ArchiveFileName $AllowAccessFromWebToFollowingIPAddresses $HTMLHeadSection $HTMLEndSection $LinksToWhoIs $LinksToIPWhoIs $LogFile $LogType $LogFormat $LogSeparator $Logo $LogoLink $StyleSheet $WrapperScript $SiteDomain $UseHTTPSLinkForUrl $URLQuerySeparators $URLWithAnchor $ErrorMessages $ShowFlagLinks /; ($DirLock, $DirCgi, $DirConfig, $DirData, $DirIcons, $DirLang, $AWScript, $ArchiveFileName, $AllowAccessFromWebToFollowingIPAddresses, $HTMLHeadSection, $HTMLEndSection, $LinksToWhoIs, $LinksToIPWhoIs, $LogFile, $LogType, $LogFormat, $LogSeparator, $Logo, $LogoLink, $StyleSheet, $WrapperScript, $SiteDomain, $UseHTTPSLinkForUrl, $URLQuerySeparators, $URLWithAnchor, $ErrorMessages, $ShowFlagLinks)= ('','','','','','','','','','','','','','','','','','','','','','','','','','',''); use vars qw/ $color_Background $color_TableBG $color_TableBGRowTitle $color_TableBGTitle $color_TableBorder $color_TableRowTitle $color_TableTitle $color_text $color_textpercent $color_titletext $color_weekend $color_link $color_hover $color_other $color_h $color_k $color_p $color_e $color_x $color_s $color_u $color_v /; ($color_Background, $color_TableBG, $color_TableBGRowTitle, $color_TableBGTitle, $color_TableBorder, $color_TableRowTitle, $color_TableTitle, $color_text, $color_textpercent, $color_titletext, $color_weekend, $color_link, $color_hover, $color_other, $color_h, $color_k, $color_p, $color_e, $color_x, $color_s, $color_u, $color_v)= ('','','','','','','','','','','','','','','','','','','','','',''); # ---------- Init arrays -------- use vars qw/ @RobotsSearchIDOrder_list1 @RobotsSearchIDOrder_list2 @RobotsSearchIDOrder_listgen @SearchEnginesSearchIDOrder_list1 @SearchEnginesSearchIDOrder_list2 @SearchEnginesSearchIDOrder_listgen @BrowsersSearchIDOrder @OSSearchIDOrder @WordsToExtractSearchUrl @WordsToCleanSearchUrl @WormsSearchIDOrder @RobotsSearchIDOrder @SearchEnginesSearchIDOrder @_from_p @_from_h @_time_p @_time_h @_time_k @_time_nv_p @_time_nv_h @_time_nv_k @DOWIndex @fieldlib @keylist @request @date_request /; @RobotsSearchIDOrder = @SearchEnginesSearchIDOrder = (); @_from_p = @_from_h = (); @_time_p = @_time_h = @_time_k = @_time_nv_p = @_time_nv_h = @_time_nv_k = (); @DOWIndex = @fieldlib = @keylist = (); #New params OnlyUsers and OnlyLines defined by E-Lane use vars qw/ @MiscListOrder %MiscListCalc @OSFamily %BrowsersFamily @SessionsRange %SessionsAverage %LangBrowserToLangAwstats %LangAWStatsToFlagAwstats @HostAliases @AllowAccessFromWebToFollowingAuthenticatedUsers @DefaultFile @SkipDNSLookupFor @SkipHosts @SkipUserAgents @SkipFiles @OnlyHosts @OnlyUserAgents @OnlyFiles @OnlyUsers @OnlyLines @URLWithQueryWithOnly @URLWithQueryWithout @ExtraName @ExtraCondition @ExtraStatTypes @MaxNbOfExtra @MinHitExtra @ExtraFirstColumnTitle @ExtraFirstColumnValues @ExtraFirstColumnFormat @ExtraCodeFilter @ExtraConditionType @ExtraConditionTypeVal @ExtraFirstColumnValuesType @ExtraFirstColumnValuesTypeVal @ExtraAddAverageRow @ExtraAddSumRow @PluginsToLoad /; @MiscListOrder=('AddToFavourites','JavascriptDisabled','JavaEnabled','DirectorSupport','FlashSupport','RealPlayerSupport','QuickTimeSupport','WindowsMediaPlayerSupport','PDFSupport'); %MiscListCalc=('TotalMisc'=>'','AddToFavourites'=>'u','JavascriptDisabled'=>'hm','JavaEnabled'=>'hm','DirectorSupport'=>'hm','FlashSupport'=>'hm','RealPlayerSupport'=>'hm','QuickTimeSupport'=>'hm','WindowsMediaPlayerSupport'=>'hm','PDFSupport'=>'hm'); @OSFamily=('win','mac'); #%BrowsersFamily=('msie'=>1,'netscape'=>2,'mozilla'=>3); %BrowsersFamily=('msie'=>1,'netscape'=>2); @SessionsRange=('0s-30s','30s-2mn','2mn-5mn','5mn-15mn','15mn-30mn','30mn-1h','1h+'); %SessionsAverage=('0s-30s',15,'30s-2mn',75,'2mn-5mn',210,'5mn-15mn',600,'15mn-30mn',1350,'30mn-1h',2700,'1h+',3600); # HTTP-Accept or Lang parameter => AWStats code to use for lang # ISO-639-1 or 2 or other => awstats-xx.txt where xx is ISO-639-1 %LangBrowserToLangAwstats=( 'sq'=>'al','ar'=>'ar','ba'=>'ba','bg'=>'bg','zh-tw'=>'tw','zh'=>'cn','cz'=>'cz', 'de'=>'de','da'=>'dk', 'en'=>'en','et'=>'et','fi'=>'fi','fr'=>'fr','gl'=>'gl', 'es'=>'es','eu'=>'eu','ca'=>'ca', 'el'=>'gr','hu'=>'hu','is'=>'is','in'=>'id','it'=>'it', 'ja'=>'jp','ko'=>'kr','lv'=>'lv','nl'=>'nl','no'=>'nb','nb'=>'nb','nn'=>'nn', 'pl'=>'pl','pt'=>'pt','pt-br'=>'br','ro'=>'ro','ru'=>'ru','sr'=>'sr','sk'=>'sk', 'sv'=>'se','th'=>'th','tr'=>'tr','uk'=>'ua','cy'=>'cy','wlk'=>'cy' ); %LangAWStatsToFlagAwstats=( # If flag (country ISO-3166 two letters) is not same than AWStats Lang code 'ca'=>'es_cat','et'=>'ee','eu','es_eu', 'cy'=>'wlk', 'gl'=>'glg', 'he'=>'il', 'ar'=>'sa', 'sr'=>'cs' ); @HostAliases = @AllowAccessFromWebToFollowingAuthenticatedUsers=(); @DefaultFile = @SkipDNSLookupFor = (); @SkipHosts = @SkipUserAgents = @SkipFiles = (); @OnlyHosts = @OnlyUserAgents = @OnlyFiles = (); #Defined OnlyXXX by E-Lane @OnlyUsers = @OnlyLines = (); @URLWithQueryWithOnly = @URLWithQueryWithout = (); @ExtraName = @ExtraCondition = @ExtraStatTypes = @MaxNbOfExtra = @MinHitExtra = (); @ExtraFirstColumnTitle = @ExtraFirstColumnValues = @ExtraFirstColumnFormat = (); @ExtraCodeFilter = @ExtraConditionType = @ExtraConditionTypeVal = (); @ExtraFirstColumnValuesType = @ExtraFirstColumnValuesTypeVal = (); @ExtraAddAverageRow = @ExtraAddSumRow = (); @PluginsToLoad = (); # ---------- Init hash arrays -------- use vars qw/ %BrowsersHashIDLib %BrowsersHashIcon %BrowsersHereAreGrabbers %DomainsHashIDLib %MimeHashLib %MimeHashIcon %MimeHashFamily %OSHashID %OSHashLib %RobotsHashIDLib %RobotsAffiliateLib %SearchEnginesHashID %SearchEnginesHashLib %SearchEnginesKnownUrl %NotSearchEnginesKeys %WormsHashID %WormsHashLib %WormsHashTarget /; use vars qw/ %HTMLOutput %NoLoadPlugin %FilterIn %FilterEx %BadFormatWarning %MonthNumLib %ValidHTTPCodes %ValidSMTPCodes %TrapInfosForHTTPErrorCodes %NotPageList %DayBytes %DayHits %DayPages %DayVisits %MaxNbOf %MinHit %ListOfYears %HistoryAlreadyFlushed %PosInFile %ValueInFile %val %nextval %egal %TmpDNSLookup %TmpOS %TmpRefererServer %TmpRobot %TmpBrowser %MyDNSTable /; %HTMLOutput = %NoLoadPlugin = %FilterIn = %FilterEx = (); %BadFormatWarning = (); %MonthNumLib = (); %ValidHTTPCodes = %ValidSMTPCodes = (); %TrapInfosForHTTPErrorCodes=(); $TrapInfosForHTTPErrorCodes{404}=1; # TODO Add this in config file %NotPageList=(); %DayBytes = %DayHits = %DayPages = %DayVisits = (); %MaxNbOf = %MinHit = (); %ListOfYears = %HistoryAlreadyFlushed = %PosInFile = %ValueInFile = (); %val = %nextval = %egal = (); %TmpDNSLookup = %TmpOS = %TmpRefererServer = %TmpRobot = %TmpBrowser = %MyDNSTable = (); use vars qw/ %FirstTime %LastTime %MonthHostsKnown %MonthHostsUnknown %MonthUnique %MonthVisits %MonthPages %MonthHits %MonthBytes %MonthNotViewedPages %MonthNotViewedHits %MonthNotViewedBytes %_session %_browser_h %_domener_p %_domener_h %_domener_k %_errors_h %_errors_k %_filetypes_h %_filetypes_k %_filetypes_gz_in %_filetypes_gz_out %_host_p %_host_h %_host_k %_host_l %_host_s %_host_u %_waithost_e %_waithost_l %_waithost_s %_waithost_u %_keyphrases %_keywords %_os_h %_pagesrefs_p %_pagesrefs_h %_robot_h %_robot_k %_robot_l %_robot_r %_worm_h %_worm_k %_worm_l %_login_h %_login_p %_login_k %_login_l %_screensize_h %_misc_p %_misc_h %_misc_k %_cluster_p %_cluster_h %_cluster_k %_se_referrals_p %_se_referrals_h %_sider404_h %_referer404_h %_url_p %_url_k %_url_e %_url_x %_unknownreferer_l %_unknownrefererbrowser_l %_emails_h %_emails_k %_emails_l %_emailr_h %_emailr_k %_emailr_l /; &Init_HashArray(); # ---------- Init Regex -------- use vars qw/ $regclean1 $regclean2 $regdate /; $regclean1=qr/<(recnb|\/td)>/i; $regclean2=qr/<\/?[^<>]+>/i; $regdate=qr/(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)/; # ---------- Init Tie::hash arrays -------- # Didn't find a tie that increase speed #use Tie::StdHash; #use Tie::Cache::LRU; #tie %_host_p, 'Tie::StdHash'; #tie %TmpOS, 'Tie::Cache::LRU'; # PROTOCOL CODES use vars qw/ %httpcodelib %ftpcodelib %smtpcodelib /; # DEFAULT MESSAGE use vars qw/ @Message /; @Message=( 'Unknown', 'Unknown (unresolved ip)', 'Others', 'View details', 'Day', 'Month', 'Year', 'Statistics for', 'First visit', 'Last visit', 'Number of visits', 'Unique visitors', 'Visit', 'different keywords', 'Search', 'Percent', 'Traffic', 'Domains/Countries', 'Visitors', 'Pages-URL', 'Hours', 'Browsers', '', 'Referers', 'Never updated (See \'Build/Update\' on awstats_setup.html page)', 'Visitors domains/countries', 'hosts', 'pages', 'different pages-url', 'Viewed', 'Other words', 'Pages not found', 'HTTP Error codes', 'Netscape versions', 'IE versions', 'Last Update', 'Connect to site from', 'Origin', 'Direct address / Bookmarks', 'Origin unknown', 'Links from an Internet Search Engine', 'Links from an external page (other web sites except search engines)', 'Links from an internal page (other page on same site)', 'Keyphrases used on search engines', 'Keywords used on search engines', 'Unresolved IP Address', 'Unknown OS (Referer field)', 'Required but not found URLs (HTTP code 404)', 'IP Address', 'Error Hits', 'Unknown browsers (Referer field)', 'different robots', 'visits/visitor', 'Robots/Spiders visitors', 'Free realtime logfile analyzer for advanced web statistics', 'of', 'Pages', 'Hits', 'Versions', 'Operating Systems', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec', 'Navigation', 'File type', 'Update now', 'Bandwidth', 'Back to main page', 'Top', 'dd mmm yyyy - HH:MM', 'Filter', 'Full list', 'Hosts', 'Known', 'Robots', 'Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Days of week', 'Who', 'When', 'Authenticated users', 'Min', 'Average', 'Max', 'Web compression', 'Bandwidth saved', 'Compression on', 'Compression result', 'Total', 'different keyphrases', 'Entry', 'Code', 'Average size', 'Links from a NewsGroup', 'KB', 'MB', 'GB', 'Grabber', 'Yes', 'No', 'Info.', 'OK', 'Exit', 'Visits duration', 'Close window', 'Bytes', 'Search Keyphrases', 'Search Keywords', 'different refering search engines', 'different refering sites', 'Other phrases', 'Other logins (and/or anonymous users)', 'Refering search engines', 'Refering sites', 'Summary', 'Exact value not available in "Year" view', 'Data value arrays', 'Sender EMail', 'Receiver EMail', 'Reported period', 'Extra/Marketing', 'Screen sizes', 'Worm/Virus attacks', 'Add to favorites (estimated)', 'Days of month', 'Miscellaneous', 'Browsers with Java support', 'Browsers with Macromedia Director Support', 'Browsers with Flash Support', 'Browsers with Real audio playing support', 'Browsers with Quictime audio playing support', 'Browsers with Windows Media audio playing support', 'Browsers with PDF support', 'SMTP Error codes', 'Countries', 'Mails', 'Size', 'First', 'Last', 'Exclude filter', 'Codes shown here gave hits or traffic "not viewed" by visitors, so they are not included in other charts.', 'Cluster', 'Robots shown here gave hits or traffic "not viewed" by visitors, so they are not included in other charts.', 'Numbers after + are successful hits on "robots.txt" files', 'Worms shown here gave hits or traffic "not viewed" by visitors, so thay are not included in other charts.', 'Not viewed traffic includes traffic generated by robots, worms, or replies with special HTTP status codes.', 'Traffic viewed', 'Traffic not viewed', 'Monthly history', 'Worms', 'different worms', 'Mails successfully sent', 'Mails failed/refused', 'Sensitive targets', 'Javascript disabled' ); #------------------------------------------------------------------------------ # Functions #------------------------------------------------------------------------------ #------------------------------------------------------------------------------ # Function: Write on ouput header of HTTP answer # Parameters: None # Input: $HeaderHTTPSent $BuildReportFormat $PageCode $Expires # Output: $HeaderHTTPSent=1 # Return: None #------------------------------------------------------------------------------ sub http_head { if (! $HeaderHTTPSent) { if ($BuildReportFormat eq 'xhtml' || $BuildReportFormat eq 'xml') { print ($ENV{'HTTP_USER_AGENT'}=~/MSIE|Googlebot/i?"Content-type: text/html; charset=$PageCode\n":"Content-type: text/xml; charset=$PageCode\n"); } else { print "Content-type: text/html; charset=$PageCode\n"; } # Expires must be GMT ANSI asctime and must be after Content-type to avoid pb with some servers (SAMBAR) if ($Expires =~ /^\d+$/) { print "Cache-Control: public\n"; print "Last-Modified: ".gmtime($starttime)."\n"; print "Expires: ".(gmtime($starttime+$Expires))."\n"; } print "\n"; } $HeaderHTTPSent++;; } #------------------------------------------------------------------------------ # Function: Write on ouput header of HTML page # Parameters: None # Input: %HTMLOutput $PluginMode $Expires $Lang $StyleSheet $HTMLHeadSection $PageCode $PageDir # Output: $HeaderHTMLSent=1 # Return: None #------------------------------------------------------------------------------ sub html_head { my $dir=$PageDir?'right':'left'; if (scalar keys %HTMLOutput || $PluginMode) { my $MetaRobot=0; # meta robots my $periodtitle=" ($YearRequired".($MonthRequired ne 'all'?"-$MonthRequired":"").")"; # Write head section if ($BuildReportFormat eq 'xhtml' || $BuildReportFormat eq 'xml') { if ($PageCode) { print "\n"; } else { print "\n"; }; if ($FrameName ne 'index') { print "\n"; } else { print "\n"; } print "\n"; } else { if ($FrameName ne 'index') { print "\n\n"; } else { print "\n\n"; } print "\n"; } print "\n"; print "\n"; if ($MetaRobot) { print "\n"; } else { print "\n"; } # Affiche tag meta content-type if ($BuildReportFormat eq 'xhtml' || $BuildReportFormat eq 'xml') { print ($ENV{'HTTP_USER_AGENT'}=~/MSIE|Googlebot/i?"\n":"\n"); } else { print "\n"; } if ($Expires) { print "\n"; } print "\n"; if ($MetaRobot && $FrameName ne 'mainleft') { print "\n"; } print "$Message[7] $SiteDomain$periodtitle\n"; if ($FrameName ne 'index') { # A STYLE section must be in head section. Do not use " for number in a style section print "\n"; if ($StyleSheet) { print "\n"; } } print "\n\n"; if ($FrameName ne 'index') { print "\n"; } } $HeaderHTMLSent++; } #------------------------------------------------------------------------------ # Function: Write on ouput end of HTML page # Parameters: 0|1 (0=no list plugins,1=list plugins) # Input: %HTMLOutput $HTMLEndSection $FrameName $BuildReportFormat # Output: None # Return: None #------------------------------------------------------------------------------ sub html_end { my $listplugins=shift||0; if (scalar keys %HTMLOutput) { # Call to plugins' function AddHTMLBodyFooter foreach my $pluginname (keys %{$PluginsLoaded{'AddHTMLBodyFooter'}}) { my $function="AddHTMLBodyFooter_$pluginname()"; eval("$function"); } if ($FrameName ne 'index' && $FrameName ne 'mainleft') { print "$Center

\n"; print ""; print "Advanced Web Statistics $VERSION - Created by $PROG"; if ($listplugins) { my $atleastoneplugin=0; foreach my $pluginname (keys %{$PluginsLoaded{'init'}}) { if (! $atleastoneplugin) { $atleastoneplugin=1; print " (with plugin "; } else { print ", "; } print "$pluginname"; } if ($atleastoneplugin) { print ")"; } } print "
\n"; if ($HTMLEndSection) { print "
\n$HTMLEndSection\n"; } } print "\n"; if ($FrameName ne 'index') { if ($FrameName ne 'mainleft' && $BuildReportFormat eq 'html') { print "
\n"; } print "\n"; } print "\n"; # print "\n"; } } #------------------------------------------------------------------------------ # Function: Print on stdout tab header of a chart # Parameters: $title $tooltipnb [$width percentage of chart title] # Input: None # Output: None # Return: None #------------------------------------------------------------------------------ sub tab_head { my $title=shift; my $tooltipnb=shift; my $width=shift||70; my $class=shift; if ($width == 70 && $QueryString =~ /buildpdf/i) { print "\n"; } else { print "
\n"; } if ($tooltipnb) { print ""; } else { print ""; } print "\n"; print "
$title
$title  
\n"; if ($width == 70 && $QueryString =~ /buildpdf/i) { print "\n"; } else { print "
\n"; } } #------------------------------------------------------------------------------ # Function: Print on stdout tab ender of a chart # Parameters: None # Input: None # Output: None # Return: None #------------------------------------------------------------------------------ sub tab_end { my $string=shift; print "
"; if ($string) { print "$string
\n"; } print "
\n\n"; } #------------------------------------------------------------------------------ # Function: Write error message and exit # Parameters: $message $secondmessage $thirdmessage $donotshowsetupinfo # Input: $HeaderHTTPSent $HeaderHTMLSent %HTMLOutput $LogSeparator $LogFormat # Output: None # Return: None #------------------------------------------------------------------------------ sub error { my $message=shift||''; if (scalar keys %HTMLOutput) { $message =~ s/\/>/g; } my $secondmessage=shift||''; my $thirdmessage=shift||''; my $donotshowsetupinfo=shift||0; if (! $HeaderHTTPSent && $ENV{'GATEWAY_INTERFACE'}) { http_head(); } if (! $HeaderHTMLSent && scalar keys %HTMLOutput) { print "\n"; $HeaderHTMLSent=1; } if ($Debug) { debug("$message $secondmessage $thirdmessage",1); } my $tagbold=''; my $tagunbold=''; my $tagbr=''; my $tagfontred=''; my $tagfontgrey=''; my $tagunfont=''; if (scalar keys %HTMLOutput) { $tagbold=''; $tagunbold=''; $tagbr='
'; $tagfontred=''; $tagfontgrey=''; $tagunfont=''; } if (! $ErrorMessages && $message =~ /^Format error$/i) { # Files seems to have bad format if (scalar keys %HTMLOutput) { print "

\n"; } if ($message !~ $LogSeparator) { # Bad LogSeparator parameter print "${tagfontred}AWStats did not found the ${tagbold}LogSeparator${tagunbold} in your log records.${tagbr}${tagunfont}\n"; } else { # Bad LogFormat parameter print "AWStats did not find any valid log lines that match your ${tagbold}LogFormat${tagunbold} parameter, in the ${NbOfLinesForCorruptedLog}th first non commented lines read of your log.${tagbr}\n"; print "${tagfontred}Your log file ${tagbold}$thirdmessage${tagunbold} must have a bad format or ${tagbold}LogFormat${tagunbold} parameter setup does not match this format.${tagbr}${tagbr}${tagunfont}\n"; print "Your AWStats ${tagbold}LogFormat${tagunbold} parameter is:\n"; print "${tagbold}$LogFormat${tagunbold}${tagbr}\n"; print "This means each line in your web server log file need to have "; if ($LogFormat == 1) { print "${tagbold}\"combined log format\"${tagunbold} like this:${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "111.22.33.44 - - [10/Jan/2001:02:14:14 +0200] \"GET / HTTP/1.1\" 200 1234 \"http://www.fromserver.com/from.htm\" \"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)\"\n"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}\n":""); } if ($LogFormat == 2) { print "${tagbold}\"MSIE Extended W3C log format\"${tagunbold} like this:${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "date time c-ip c-username cs-method cs-uri-sterm sc-status sc-bytes cs-version cs(User-Agent) cs(Referer)\n"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}\n":""); } if ($LogFormat == 3) { print "${tagbold}\"WebStar native log format\"${tagunbold}${tagbr}\n"; } if ($LogFormat == 4) { print "${tagbold}\"common log format\"${tagunbold} like this:${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "111.22.33.44 - - [10/Jan/2001:02:14:14 +0200] \"GET / HTTP/1.1\" 200 1234\n"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}\n":""); } if ($LogFormat == 5) { print "${tagbold}\"ISA native log format\"${tagunbold}${tagbr}\n"; } if ($LogFormat == 6) { print "${tagbold}\"Lotus Notes/Lotus Domino\"${tagunbold}${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "111.22.33.44 - Firstname Middlename Lastname [10/Jan/2001:02:14:14 +0200] \"GET / HTTP/1.1\" 200 1234 \"http://www.fromserver.com/from.htm\" \"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)\"\n"; print (scalar keys %HTMLOutput?"${tagbr}${tagbr}\n":""); } if ($LogFormat !~ /^[1-6]$/) { print "the following personalized log format:${tagbr}\n"; print (scalar keys %HTMLOutput?"$tagfontgrey":""); print "$LogFormat\n"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}\n":""); } print "And this is an example of records AWStats found in your log file (the record number $NbOfLinesForCorruptedLog in your log):\n"; print (scalar keys %HTMLOutput?"
$tagfontgrey":""); print "$secondmessage"; print (scalar keys %HTMLOutput?"$tagunfont${tagbr}${tagbr}":""); print "\n"; } #print "Note: If your $NbOfLinesForCorruptedLog first lines in your log files are wrong because of "; #print "a worm virus attack, you can increase the NbOfLinesForCorruptedLog parameter in config file.\n"; #print "\n"; } else { print (scalar keys %HTMLOutput?"
$tagfontred\n":""); print ($ErrorMessages?"$ErrorMessages":"Error: $message"); print (scalar keys %HTMLOutput?"\n
":""); print "\n"; } if (! $ErrorMessages && ! $donotshowsetupinfo) { if ($message =~ /Couldn.t open config file/i) { my $dir=$DIR; if ($dir =~ /^\./) { $dir.='/../..'; } else { $dir =~ s/[\\\/]?wwwroot[\/\\]cgi-bin[\\\/]?//; } print "${tagbr}\n"; if ($ENV{'GATEWAY_INTERFACE'}) { print "- ${tagbold}Did you use the correct URL ?${tagunbold}${tagbr}\n"; print "Example: http://localhost/awstats/awstats.pl?config=mysite${tagbr}\n"; print "Example: http://127.0.0.1/cgi-bin/awstats.pl?config=mysite${tagbr}\n"; } else { print "- ${tagbold}Did you use correct config parameter ?${tagunbold}${tagbr}\n"; print "Example: If your config file is awstats.mysite.conf, use -config=mysite\n"; } print "- ${tagbold}Did you create your config file 'awstats.$SiteConfig.conf' ?${tagunbold}${tagbr}\n"; print "If not, you can run \"$dir/tools/configure.pl\"\nfrom command line, or create it manually.${tagbr}\n"; print "${tagbr}\n"; } else { print "${tagbr}${tagbold}Setup (".($FileConfig?"'".$FileConfig."'":"Config")." file, web server or permissions) may be wrong.${tagunbold}${tagbr}\n"; } print "Check config file, permissions and AWStats documentation (in 'docs' directory).\n"; } # Remove lock if not a lock message if ($EnableLockForUpdate && $message !~ /lock file/) { &Lock_Update(0); } if (scalar keys %HTMLOutput) { print "\n"; } exit 1; } #------------------------------------------------------------------------------ # Function: Write a warning message # Parameters: $message # Input: $HeaderHTTPSent $HeaderHTMLSent $WarningMessage %HTMLOutput # Output: None # Return: None #------------------------------------------------------------------------------ sub warning { my $messagestring=shift; if (! $HeaderHTTPSent && $ENV{'GATEWAY_INTERFACE'}) { http_head(); } if (! $HeaderHTMLSent) { html_head(); } if ($Debug) { debug("$messagestring",1); } if ($WarningMessages) { if (scalar keys %HTMLOutput) { $messagestring =~ s/\n/\/g; print "$messagestring
\n"; } else { print "$messagestring\n"; } } } #------------------------------------------------------------------------------ # Function: Write debug message and exit # Parameters: $string $level # Input: %HTMLOutput $Debug=required level $DEBUGFORCED=required level forced # Output: None # Return: None #------------------------------------------------------------------------------ sub debug { my $level = $_[1] || 1; if (! $HeaderHTTPSent && $ENV{'GATEWAY_INTERFACE'}) { http_head(); } # To send the HTTP header and see debug if ($level <= $DEBUGFORCED) { my $debugstring = $_[0]; if (! $DebugResetDone) { open(DEBUGFORCEDFILE,"debug.log"); close DEBUGFORCEDFILE; chmod 0666,"debug.log"; $DebugResetDone=1; } open(DEBUGFORCEDFILE,">>debug.log"); print DEBUGFORCEDFILE localtime(time)." - $$ - DEBUG $level - $debugstring\n"; close DEBUGFORCEDFILE; } if ($DebugMessages && $level <= $Debug) { my $debugstring = $_[0]; if (scalar keys %HTMLOutput) { $debugstring =~ s/^ /   /; $debugstring .= "
"; } print localtime(time)." - DEBUG $level - $debugstring\n"; } } #------------------------------------------------------------------------------ # Function: Optimize an array removing duplicate entries # Parameters: @Array notcasesensitive=0|1 # Input: None # Output: None # Return: None #------------------------------------------------------------------------------ sub OptimizeArray { my $array=shift; my @arrayunreg=map{if (/\(\?[-\w]*:(.*)\)/) { $1 } } @$array; my $notcasesensitive=shift; my $searchlist=0; if ($Debug) { debug("OptimizeArray (notcasesensitive=$notcasesensitive)",4); } while ($searchlist>-1 && @arrayunreg) { my $elemtoremove=-1; OPTIMIZELOOP: foreach my $i ($searchlist..(scalar @arrayunreg)-1) { # Search if $i elem is already treated by another elem foreach my $j (0..(scalar @arrayunreg)-1) { if ($i == $j) { next; } my $parami=$notcasesensitive?lc($arrayunreg[$i]):$arrayunreg[$i]; my $paramj=$notcasesensitive?lc($arrayunreg[$j]):$arrayunreg[$j]; if ($Debug) { debug(" Compare $i ($parami) to $j ($paramj)",4); } if (index($parami,$paramj)>-1) { if ($Debug) { debug(" Elem $i ($arrayunreg[$i]) already treated with elem $j ($arrayunreg[$j])",4); } $elemtoremove=$i; last OPTIMIZELOOP; } } } if ($elemtoremove > -1) { if ($Debug) { debug(" Remove elem $elemtoremove - $arrayunreg[$elemtoremove]",4); } splice @arrayunreg, $elemtoremove, 1; $searchlist=$elemtoremove; } else { $searchlist=-1; } } if ($notcasesensitive) { return map{qr/$_/i} @arrayunreg; } return map{qr/$_/} @arrayunreg; } #------------------------------------------------------------------------------ # Function: Check if parameter is in SkipDNSLookupFor array # Parameters: ip @SkipDNSLookupFor (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub SkipDNSLookup { foreach (@SkipDNSLookupFor) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @SkipDNSLookupFor } #------------------------------------------------------------------------------ # Function: Check if parameter is in SkipHosts array # Parameters: host @SkipHosts (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub SkipHost { foreach (@SkipHosts) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @SkipHosts } #------------------------------------------------------------------------------ # Function: Check if parameter is in SkipUserAgents array # Parameters: useragent @SkipUserAgents (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub SkipUserAgent { foreach (@SkipUserAgents) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @SkipUserAgent } #------------------------------------------------------------------------------ # Function: Check if parameter is in SkipFiles array # Parameters: url @SkipFiles (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub SkipFile { foreach (@SkipFiles) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @SkipFiles } #------------------------------------------------------------------------------ # Function: Check if parameter is in OnlyHosts array # Parameters: host @OnlyHosts (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub OnlyHost { foreach (@OnlyHosts) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @OnlyHosts } #------------------------------------------------------------------------------ # Function: Check if parameter is in OnlyUserAgents array # Parameters: useragent @OnlyUserAgents (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub OnlyUserAgent { foreach (@OnlyUserAgents) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @OnlyHosts } #------------------------------------------------------------------------------ # Function: Check if parameter is in OnlyUsers array # Parameters: useragent @OnlyUsers (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found # Author: E-Lane #------------------------------------------------------------------------------ sub OnlyUser { foreach (@OnlyUsers) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @OnlyUsers } #------------------------------------------------------------------------------ # Function: Check if parameter is in OnlyLines array # Parameters: useragent @OnlyLines (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found # Author: E-Lane #------------------------------------------------------------------------------ sub OnlyLine { foreach (@OnlyLines) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @OnlyUsers } #------------------------------------------------------------------------------ # Function: Check if parameter is in OnlyFiles array # Parameters: url @OnlyFiles (a NOT case sensitive precompiled regex array) # Return: 0 Not found, 1 Found #------------------------------------------------------------------------------ sub OnlyFile { foreach (@OnlyFiles) { if ($_[0] =~ /$_/) { return 1; } } 0; # Not in @OnlyFiles } #------------------------------------------------------------------------------ # Function: Return day of week of a day # Parameters: $day $month $year # Return: 0-6 #------------------------------------------------------------------------------ sub DayOfWeek { my ($day, $month, $year) = @_; if ($Debug) { debug("DayOfWeek for $day $month $year",4); } if ($month < 3) { $month += 10; $year--; } else { $month -= 2; } my $cent = sprintf("%1i",($year/100)); my $y = ($year % 100); my $dw = (sprintf("%1i",(2.6*$month)-0.2) + $day + $y + sprintf("%1i",($y/4)) + sprintf("%1i",($cent/4)) - (2*$cent)) % 7; $dw += 7 if ($dw<0); if ($Debug) { debug(" is $dw",4); } return $dw; } #------------------------------------------------------------------------------ # Function: Return 1 if a date exists # Parameters: $day $month $year # Return: 1 if date exists else 0 #------------------------------------------------------------------------------ sub DateIsValid { my ($day, $month, $year) = @_; if ($Debug) { debug("DateIsValid for $day $month $year",4); } if ($day < 1) { return 0; } if ($day > 31) { return 0; } if ($month==4 || $month==6 || $month==9 || $month==11) { if ($day > 30) { return 0; } } elsif ($month==2) { my $leapyear=($year%4==0?1:0); # A leap year every 4 years if ($year%100==0 && $year%400!=0) { $leapyear=0; } # Except if year is 100x and not 400x if ($day > (28+$leapyear)) { return 0; } } return 1; } #------------------------------------------------------------------------------ # Function: Return string of visit duration # Parameters: $starttime $endtime # Input: None # Output: None # Return: A string that identify the visit duration range #------------------------------------------------------------------------------ sub GetSessionRange { my $starttime = my $endtime; if (shift =~ /$regdate/o) { $starttime = Time::Local::timelocal($6,$5,$4,$3,$2-1,$1); } if (shift =~ /$regdate/o) { $endtime = Time::Local::timelocal($6,$5,$4,$3,$2-1,$1); } my $delay=$endtime-$starttime; if ($Debug) { debug("GetSessionRange $endtime - $starttime = $delay",4); } if ($delay <= 30) { return $SessionsRange[0]; } if ($delay <= 120) { return $SessionsRange[1]; } if ($delay <= 300) { return $SessionsRange[2]; } if ($delay <= 900) { return $SessionsRange[3]; } if ($delay <= 1800) { return $SessionsRange[4]; } if ($delay <= 3600) { return $SessionsRange[5]; } return $SessionsRange[6]; } #------------------------------------------------------------------------------ # Function: Read config file # Parameters: None or configdir to scan # Input: $DIR $PROG $SiteConfig # Output: Global variables # Return: - #------------------------------------------------------------------------------ sub Read_Config { # Check config file in common possible directories : # Windows : "$DIR" (same dir than awstats.pl) # Standard, Mandrake and Debian package : "/etc/awstats" # Other possible directories : "/usr/local/etc/awstats", "/etc" # FHS standard, Suse package : "/etc/opt/awstats" my $configdir=shift; my @PossibleConfigDir=(); if ($configdir) { @PossibleConfigDir=("$configdir","$DIR","${DIR}/../../../config","/etc/awstats","/usr/local/etc/awstats","/etc","/etc/opt/awstats"); } else { @PossibleConfigDir=("$DIR","${DIR}/../../../config","/etc/awstats","/usr/local/etc/awstats","/etc","/etc/opt/awstats"); } # Open config file $FileConfig=''; # $FileConfig=$FileSuffix=''; foreach (@PossibleConfigDir) { my $searchdir=$_; if ($searchdir && $searchdir !~ /[\\\/]$/) { $searchdir .= "/"; } if (open(CONFIG,"$searchdir$PROG.$SiteConfig.conf")) { $FileConfig="$searchdir$PROG.$SiteConfig.conf"; $FileSuffix=".$SiteConfig$FileSuffix"; last; } if (open(CONFIG,"$searchdir$PROG.conf")) { $FileConfig="$searchdir$PROG.conf"; $FileSuffix="$FileSuffix"; last; } } if (! $FileConfig) { error("Couldn't open config file \"$PROG.$SiteConfig.conf\" nor \"$PROG.conf\" after searching in path \"".join(',',@PossibleConfigDir)."\": $!"); } # Analyze config file content and close it &Parse_Config( *CONFIG , 1 , $FileConfig); close CONFIG; # If parameter NotPageList not found, init for backward compatibility if (! $FoundNotPageList) { %NotPageList=('css'=>1,'js'=>1,'class'=>1,'gif'=>1,'jpg'=>1,'jpeg'=>1,'png'=>1,'bmp'=>1); } # If parameter ValidHTTPCodes empty, init for backward compatibility if (! scalar keys %ValidHTTPCodes) { $ValidHTTPCodes{"200"}=$ValidHTTPCodes{"304"}=1; } # If parameter ValidSMTPCodes empty, init for backward compatibility if (! scalar keys %ValidSMTPCodes) { $ValidSMTPCodes{"1"}=$ValidSMTPCodes{"250"}=1; } } #------------------------------------------------------------------------------ # Function: Parse content of a config file # Parameters: opened file handle, depth level, file name # Input: - # Output: Global variables # Return: - #------------------------------------------------------------------------------ sub Parse_Config { my ( $confighandle ) = $_[0]; my $level = $_[1]; my $configFile = $_[2]; my $versionnum=0; my $conflinenb=0; if ($level > 10) { error("$PROG can't read down more than 10 level of includes. Check that no 'included' config files include their parent config file (this cause infinite loop)."); } while (<$confighandle>) { chomp $_; s/\r//; $conflinenb++; # Extract version from first line if (! $versionnum && $_ =~ /^# AWSTATS CONFIGURE FILE (\d+).(\d+)/i) { $versionnum=($1*1000)+$2; #if ($Debug) { debug(" Configure file version is $versionnum",1); } next; } if ($_ =~ /^\s*$/) { next; } # Check includes if ($_ =~ /^Include "([^\"]+)"/ || $_ =~ /^#include "([^\"]+)"/) { # #include kept for backward compatibility my $includeFile = $1; if ($Debug) { debug("Found an include : $includeFile",2); } if ( $includeFile !~ /^[\\\/]/ ) { # Correct relative include files if ($FileConfig =~ /^(.*[\\\/])[^\\\/]*$/) { $includeFile = "$1$includeFile"; } } if ($level > 1) { warning("Warning: Perl versions before 5.6 cannot handle nested includes"); next; } if ( open( CONFIG_INCLUDE, $includeFile ) ) { &Parse_Config( *CONFIG_INCLUDE , $level+1, $includeFile); close( CONFIG_INCLUDE ); } else { error("Could not open include file: $includeFile" ); } next; } # Remove comments if ($_ =~ /^\s*#/) { next; } $_ =~ s/\s#.*$//; # Extract param and value my ($param,$value)=split(/=/,$_,2); $param =~ s/^\s+//; $param =~ s/\s+$//; # If not a param=value, try with next line if (! $param) { warning("Warning: Syntax error line $conflinenb in file '$configFile'. Config line is ignored."); next; } if (! defined $value) { warning("Warning: Syntax error line $conflinenb in file '$configFile'. Config line is ignored."); next; } if ($value) { $value =~ s/^\s+//; $value =~ s/\s+$//; $value =~ s/^\"//; $value =~ s/\";?$//; # Replace __MONENV__ with value of environnement variable MONENV # Must be able to replace __VAR_1____VAR_2__ while ($value =~ /__([^\s_]+(?:_[^\s_]+)*)__/) { my $var=$1; $value =~ s/__${var}__/$ENV{$var}/g; } } # Initialize parameter for (param,value) if ($param =~ /^LogFile/) { if ($QueryString !~ /logfile=([^\s&]+)/i) { $LogFile=$value; } next; } if ($param =~ /^DirIcons/) { if ($QueryString !~ /diricons=([^\s&]+)/i) { $DirIcons=$value; } next; } #We have modified this option, because is more important the line command param if ($param =~ /^SiteDomain/) { # No regex test as SiteDomain is always exact value if ($SiteDomain eq '') { $SiteDomain=$value;} next; } if ($param =~ /^HostAliases/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ s/^\@//) { # If list of hostaliases in a file open(DATAFILE,"<$elem") || error("Failed to open file '$elem' declared in HostAliases parameter"); my @val=map(/^(.*)$/i,); push @HostAliases, map{qr/^$_$/i} @val; close(DATAFILE); } else { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @HostAliases, qr/$elem/i; } } } next; } # Special optional setup params if ($param =~ /^SkipDNSLookupFor/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @SkipDNSLookupFor, qr/$elem/i; } } next; } if ($param =~ /^AllowAccessFromWebToFollowingAuthenticatedUsers/) { foreach (split(/\s+/,$value)) { push @AllowAccessFromWebToFollowingAuthenticatedUsers,$_; } next; } if ($param =~ /^DefaultFile/) { foreach my $elem (split(/\s+/,$value)) { # No REGEX for this option #if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } #else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @DefaultFile,$elem; } } next; } if ($param =~ /^SkipHosts/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @SkipHosts, qr/$elem/i; } } next; } if ($param =~ /^SkipUserAgents/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @SkipUserAgents, qr/$elem/i; } } next; } if ($param =~ /^SkipFiles/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @SkipFiles, qr/$elem/i; } } next; } if ($param =~ /^OnlyHosts/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyHosts, qr/$elem/i; } } next; } if ($param =~ /^OnlyUserAgents/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyUserAgents, qr/$elem/i; } } next; } if ($param =~ /^OnlyFiles/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyFiles, qr/$elem/i; } } next; } #Params OnlyXXX defined by E-Lane if ($param =~ /^OnlyUsers/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyUsers, qr/$elem/i; } } next; } if ($param =~ /^OnlyLines/) { foreach my $elem (split(/\s+/,$value)) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; } else { $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyLines, qr/$elem/i; } } next; } if ($param =~ /^NotPageList/) { foreach (split(/\s+/,$value)) { $NotPageList{$_}=1; } $FoundNotPageList=1; next; } if ($param =~ /^ValidHTTPCodes/) { foreach (split(/\s+/,$value)) { $ValidHTTPCodes{$_}=1; } next; } if ($param =~ /^ValidSMTPCodes/) { foreach (split(/\s+/,$value)) { $ValidSMTPCodes{$_}=1; } next; } if ($param =~ /^URLWithQueryWithOnlyFollowingParameters$/) { @URLWithQueryWithOnly=split(/\s+/,$value); next; } if ($param =~ /^URLWithQueryWithoutFollowingParameters$/) { @URLWithQueryWithout=split(/\s+/,$value); next; } # Extra parameters if ($param =~ /^ExtraSectionName(\d+)/) { $ExtraName[$1]=$value; next; } if ($param =~ /^ExtraSectionCodeFilter(\d+)/) { @{$ExtraCodeFilter[$1]}=split(/\s+/,$value); next; } if ($param =~ /^ExtraSectionCondition(\d+)/) { $ExtraCondition[$1]=$value; next; } if ($param =~ /^ExtraSectionStatTypes(\d+)/) { $ExtraStatTypes[$1]=$value; next; } if ($param =~ /^ExtraSectionFirstColumnTitle(\d+)/) { $ExtraFirstColumnTitle[$1]=$value; next; } if ($param =~ /^ExtraSectionFirstColumnValues(\d+)/) { $ExtraFirstColumnValues[$1]=$value; next; } if ($param =~ /^ExtraSectionFirstColumnFormat(\d+)/) { $ExtraFirstColumnFormat[$1]=$value; next; } if ($param =~ /^ExtraSectionAddAverageRow(\d+)/) { $ExtraAddAverageRow[$1]=$value; next; } if ($param =~ /^ExtraSectionAddSumRow(\d+)/) { $ExtraAddSumRow[$1]=$value; next; } if ($param =~ /^MaxNbOfExtra(\d+)/) { $MaxNbOfExtra[$1]=$value; next; } if ($param =~ /^MinHitExtra(\d+)/) { $MinHitExtra[$1]=$value; next; } # Special appearance parameters if ($param =~ /^LoadPlugin/) { push @PluginsToLoad, $value; next; } # Other parameter checks we need to put after MaxNbOfExtra and MinHitExtra if ($param =~ /^MaxNbOf(\w+)/) { $MaxNbOf{$1}=$value; next; } if ($param =~ /^MinHit(\w+)/) { $MinHit{$1}=$value; next; } # Check if this is a known parameter # if (! $ConfOk{$param}) { error("Unknown config parameter '$param' found line $conflinenb in file \"configFile\""); } # If parameters was not found previously, defined variable with name of param to value $$param=$value; } # For backward compatibility if ($versionnum < 5001) { $BarHeight=$BarHeight>>1; } if ($Debug) { debug("Config file read was \"$configFile\" (level $level)"); } } #------------------------------------------------------------------------------ # Function: Load the reference databases # Parameters: List of files to load # Input: $DIR # Output: Arrays and Hash tables are defined # Return: - #------------------------------------------------------------------------------ sub Read_Ref_Data { # Check lib files in common possible directories : # Windows and standard package: "$DIR/lib" (lib in same dir than awstats.pl) # Debian package : "/usr/share/awstats/lib" my @PossibleLibDir=("$DIR/lib","/usr/share/awstats/lib"); my %FilePath=(); my %DirAddedInINC=(); my @FileListToLoad=(); while (my $file=shift) { push @FileListToLoad, "$file.pm"; } if ($Debug) { debug("Call to Read_Ref_Data with files to load: ".(join(',',@FileListToLoad))); } foreach my $file (@FileListToLoad) { foreach my $dir (@PossibleLibDir) { my $searchdir=$dir; if ($searchdir && (!($searchdir =~ /\/$/)) && (!($searchdir =~ /\\$/)) ) { $searchdir .= "/"; } if (! $FilePath{$file}) { # To not load twice same file in different path if (-s "${searchdir}${file}") { $FilePath{$file}="${searchdir}${file}"; if ($Debug) { debug("Call to Read_Ref_Data [FilePath{$file}=\"$FilePath{$file}\"]"); } # Note: cygwin perl 5.8 need a push + require file if (! $DirAddedInINC{"$dir"}) { push @INC, "$dir"; $DirAddedInINC{"$dir"}=1; } my $loadret=require "$file"; #my $loadret=(require "$FilePath{$file}"||require "${file}"); } } } if (! $FilePath{$file}) { my $filetext=$file; $filetext =~ s/\.pm$//; $filetext =~ s/_/ /g; warning("Warning: Can't read file \"$file\" ($filetext detection will not work correctly).\nCheck if file is in \"".($PossibleLibDir[0])."\" directory and is readable."); } } # Sanity check (if loaded) if ((scalar keys %OSHashID) && @OSSearchIDOrder != scalar keys %OSHashID) { error("Not same number of records of OSSearchIDOrder (".(@OSSearchIDOrder)." entries) and OSHashID (".(scalar keys %OSHashID)." entries) in OS database. Check your file ".$FilePath{"operating_systems.pm"}); } if ((scalar keys %SearchEnginesHashID) && (@SearchEnginesSearchIDOrder_list1+@SearchEnginesSearchIDOrder_list2+@SearchEnginesSearchIDOrder_listgen) != scalar keys %SearchEnginesHashID) { error("Not same number of records of SearchEnginesSearchIDOrder_listx (total is ".(@SearchEnginesSearchIDOrder_list1+@SearchEnginesSearchIDOrder_list2+@SearchEnginesSearchIDOrder_listgen)." entries) and SearchEnginesHashID (".(scalar keys %SearchEnginesHashID)." entries) in Search Engines database. Check your file ".$FilePath{"search_engines.pm"}); } if ((scalar keys %BrowsersHashIDLib) && @BrowsersSearchIDOrder != (scalar keys %BrowsersHashIDLib) - 2) { error("Not same number of records of BrowsersSearchIDOrder (".(@BrowsersSearchIDOrder)." entries) and BrowsersHashIDLib (".((scalar keys %BrowsersHashIDLib) - 2)." entries without msie and netscape) in Browsers database. Check your file ".$FilePath{"browsers.pm"}); } if ((scalar keys %RobotsHashIDLib) && (@RobotsSearchIDOrder_list1+@RobotsSearchIDOrder_list2+@RobotsSearchIDOrder_listgen) != (scalar keys %RobotsHashIDLib) - 1) { error("Not same number of records of RobotsSearchIDOrder_listx (total is ".(@RobotsSearchIDOrder_list1+@RobotsSearchIDOrder_list2+@RobotsSearchIDOrder_listgen)." entries) and RobotsHashIDLib (".((scalar keys %RobotsHashIDLib) - 1)." entries without 'unknown') in Robots database. Check your file ".$FilePath{"robots.pm"}); } } #------------------------------------------------------------------------------ # Function: Get the messages for a specified language # Parameters: LanguageId # Input: $DirLang $DIR # Output: $Message table is defined in memory # Return: None #------------------------------------------------------------------------------ sub Read_Language_Data { # Check lang files in common possible directories : # Windows and standard package: "$DIR/lang" (lang in same dir than awstats.pl) # Debian package : "/usr/share/awstats/lang" my @PossibleLangDir=("$DirLang","$DIR/lang","/usr/share/awstats/lang"); my $FileLang=''; foreach (@PossibleLangDir) { my $searchdir=$_; if ($searchdir && (!($searchdir =~ /\/$/)) && (!($searchdir =~ /\\$/)) ) { $searchdir .= "/"; } if (open(LANG,"${searchdir}awstats-$_[0].txt")) { $FileLang="${searchdir}awstats-$_[0].txt"; last; } } # If file not found, we try english if (! $FileLang) { foreach (@PossibleLangDir) { my $searchdir=$_; if ($searchdir && (!($searchdir =~ /\/$/)) && (!($searchdir =~ /\\$/)) ) { $searchdir .= "/"; } if (open(LANG,"${searchdir}awstats-en.txt")) { $FileLang="${searchdir}awstats-en.txt"; last; } } } if ($Debug) { debug("Call to Read_Language_Data [FileLang=\"$FileLang\"]"); } if ($FileLang) { my $i = 0; binmode LANG; # Might avoid 'Malformed UTF-8 errors' my $cregcode=qr/^PageCode=[\t\s\"\']*([\w-]+)/i; my $cregdir=qr/^PageDir=[\t\s\"\']*([\w-]+)/i; my $cregmessage=qr/^Message\d+=/i; while () { chomp $_; s/\r//; if ($_ =~ /$cregcode/o) { $PageCode = $1; } if ($_ =~ /$cregdir/o) { $PageDir = $1; } if ($_ =~ s/$cregmessage//o) { $_ =~ s/#.*//; # Remove comments $_ =~ tr/\t / /s; # Change all blanks into " " $_ =~ s/^\s+//; $_ =~ s/\s+$//; $_ =~ s/^\"//; $_ =~ s/\"$//; $Message[$i] = $_; $i++; } } close(LANG); } else { warning("Warning: Can't find language files for \"$_[0]\". English will be used."); } # Some language string changes if ($LogType eq 'M') { # For mail $Message[8]=$Message[151]; $Message[9]=$Message[152]; $Message[57]=$Message[149]; $Message[75]=$Message[150]; } if ($LogType eq 'F') { # For web } } #------------------------------------------------------------------------------ # Function: Check if all parameters are correctly defined. If not set them to default. # Parameters: None # Input: All global variables # Output: Change on some global variables # Return: None #------------------------------------------------------------------------------ sub Check_Config { if ($Debug) { debug("Call to Check_Config"); } my %MonthNumLibEn = ("01","Jan","02","Feb","03","Mar","04","Apr","05","May","06","Jun","07","Jul","08","Aug","09","Sep","10","Oct","11","Nov","12","Dec"); # Show initial values of main parameters before check if ($Debug) { debug(" LogFile='$LogFile'",2); debug(" LogType='$LogType'",2); debug(" LogFormat='$LogFormat'",2); debug(" LogSeparator='$LogSeparator'",2); debug(" DNSLookup='$DNSLookup'",2); debug(" DirData='$DirData'",2); debug(" DirCgi='$DirCgi'",2); debug(" DirIcons='$DirIcons'",2); debug(" NotPageList ".(join(',',keys %NotPageList)),2); debug(" ValidHTTPCodes ".(join(',',keys %ValidHTTPCodes)),2); debug(" ValidSMTPCodes ".(join(',',keys %ValidSMTPCodes)),2); debug(" UseFramesWhenCGI=$UseFramesWhenCGI",2); debug(" BuildReportFormat=$BuildReportFormat",2); debug(" BuildHistoryFormat=$BuildHistoryFormat",2); debug(" URLWithQueryWithOnlyFollowingParameters=".(join(',',@URLWithQueryWithOnly)),2); debug(" URLWithQueryWithoutFollowingParameters=".(join(',',@URLWithQueryWithout)),2); } # Main section while ($LogFile =~ /%([ymdhwYMDHWNSO]+)-(\(\d+\)|\d+)/) { # Accept tag %xx-dd and %xx-(dd) my $timetag="$1"; my $timephase=quotemeta("$2"); my $timephasenb="$2"; $timephasenb=~s/[^\d]//g; if ($Debug) { debug(" Found a time tag '$timetag' with a phase of '$timephasenb' hour in log file name",1); } # Get older time my ($oldersec,$oldermin,$olderhour,$olderday,$oldermonth,$olderyear,$olderwday,$olderyday) = localtime($starttime-($timephasenb*3600)); my $olderweekofmonth=int($olderday/7); my $olderweekofyear=int(($olderyday-1+6-($olderwday==0?6:$olderwday-1))/7)+1; if ($olderweekofyear > 52) { $olderweekofyear = 1; } my $olderdaymod=$olderday%7; $olderwday++; my $olderns=Time::Local::timegm(0,0,0,$olderday,$oldermonth,$olderyear); if ($olderdaymod <= $olderwday) { if (($olderwday != 7) || ($olderdaymod != 0)) { $olderweekofmonth=$olderweekofmonth+1; } } if ($olderdaymod > $olderwday) { $olderweekofmonth=$olderweekofmonth+2; } # Change format of time variables $olderweekofmonth = "0$olderweekofmonth"; if ($olderweekofyear < 10) { $olderweekofyear = "0$olderweekofyear"; } if ($olderyear < 100) { $olderyear+=2000; } else { $olderyear+=1900; } my $oldersmallyear=$olderyear;$oldersmallyear =~ s/^..//; if (++$oldermonth < 10) { $oldermonth = "0$oldermonth"; } if ($olderday < 10) { $olderday = "0$olderday"; } if ($olderhour < 10) { $olderhour = "0$olderhour"; } if ($oldermin < 10) { $oldermin = "0$oldermin"; } if ($oldersec < 10) { $oldersec = "0$oldersec"; } # Replace tag with new value if ($timetag eq 'YYYY') { $LogFile =~ s/%YYYY-$timephase/$olderyear/ig; next; } if ($timetag eq 'YY') { $LogFile =~ s/%YY-$timephase/$oldersmallyear/ig; next; } if ($timetag eq 'MM') { $LogFile =~ s/%MM-$timephase/$oldermonth/ig; next; } if ($timetag eq 'MO') { $LogFile =~ s/%MO-$timephase/$MonthNumLibEn{$oldermonth}/ig; next; } if ($timetag eq 'DD') { $LogFile =~ s/%DD-$timephase/$olderday/ig; next; } if ($timetag eq 'HH') { $LogFile =~ s/%HH-$timephase/$olderhour/ig; next; } if ($timetag eq 'NS') { $LogFile =~ s/%NS-$timephase/$olderns/ig; next; } if ($timetag eq 'WM') { $LogFile =~ s/%WM-$timephase/$olderweekofmonth/g; next; } if ($timetag eq 'Wm') { my $olderweekofmonth0=$olderweekofmonth-1; $LogFile =~ s/%Wm-$timephase/$olderweekofmonth0/g; next; } if ($timetag eq 'WY') { $LogFile =~ s/%WY-$timephase/$olderweekofyear/g; next; } if ($timetag eq 'Wy') { my $olderweekofyear0=sprintf("%02d",$olderweekofyear-1); $LogFile =~ s/%Wy-$timephase/$olderweekofyear0/g; next; } if ($timetag eq 'DW') { $LogFile =~ s/%DW-$timephase/$olderwday/g; next; } if ($timetag eq 'Dw') { my $olderwday0=$olderwday-1; $LogFile =~ s/%Dw-$timephase/$olderwday0/g; next; } # If unknown tag error("Unknown tag '\%$timetag' in LogFile parameter."); } # Replace %YYYY %YY %MM %DD %HH with current value. Kept for backward compatibility. $LogFile =~ s/%YYYY/$nowyear/ig; $LogFile =~ s/%YY/$nowsmallyear/ig; $LogFile =~ s/%MM/$nowmonth/ig; $LogFile =~ s/%MO/$MonthNumLibEn{$nowmonth}/ig; $LogFile =~ s/%DD/$nowday/ig; $LogFile =~ s/%HH/$nowhour/ig; $LogFile =~ s/%NS/$nowns/ig; $LogFile =~ s/%WM/$nowweekofmonth/g; my $nowweekofmonth0=$nowweekofmonth-1; $LogFile =~ s/%Wm/$nowweekofmonth0/g; $LogFile =~ s/%WY/$nowweekofyear/g; my $nowweekofyear0=$nowweekofyear-1; $LogFile =~ s/%Wy/$nowweekofyear0/g; $LogFile =~ s/%DW/$nowwday/g; my $nowwday0=$nowwday-1; $LogFile =~ s/%Dw/$nowwday0/g; if (! $LogFile) { error("LogFile parameter is not defined in config/domain file"); } if ($LogType !~ /[WSMF]/i) { $LogType='W'; } $LogFormat =~ s/\\//g; if (! $LogFormat) { error("LogFormat parameter is not defined in config/domain file"); } if ($LogFormat =~ /^\d$/ && $LogFormat !~ /[1-6]/) { error("LogFormat parameter is wrong in config/domain file. Value is '$LogFormat' (should be 1,2,3,4,5 or a 'personalized AWStats log format string')"); } $LogSeparator||="\\s"; $DirData||='.'; $DirCgi||='/cgi-bin'; $DirIcons||='/icon'; if ($DNSLookup !~ /[0-2]/) { error("DNSLookup parameter is wrong in config/domain file. Value is '$DNSLookup' (should be 0 or 1)"); } if (! $SiteDomain) { error("SiteDomain parameter not defined in your config/domain file. You must add it for using this version of AWStats."); } if ($AllowToUpdateStatsFromBrowser !~ /[0-1]/) { $AllowToUpdateStatsFromBrowser=0; } if ($AllowFullYearView !~ /[0-3]/) { $AllowFullYearView=2; } # Optional setup section if ($EnableLockForUpdate !~ /[0-1]/) { $EnableLockForUpdate=0; } $DNSStaticCacheFile||='dnscache.txt'; $DNSLastUpdateCacheFile||='dnscachelastupdate.txt'; if ($DNSStaticCacheFile eq $DNSLastUpdateCacheFile) { error("DNSStaticCacheFile and DNSLastUpdateCacheFile must have different values."); } if ($AllowAccessFromWebToAuthenticatedUsersOnly !~ /[0-1]/) { $AllowAccessFromWebToAuthenticatedUsersOnly=0; } if ($CreateDirDataIfNotExists !~ /[0-1]/) { $CreateDirDataIfNotExists=0; } if ($BuildReportFormat !~ /html|xhtml|xml/i) { $BuildReportFormat='html'; } if ($BuildHistoryFormat !~ /text|xml/) { $BuildHistoryFormat='text'; } if ($SaveDatabaseFilesWithPermissionsForEveryone !~ /[0-1]/) { $SaveDatabaseFilesWithPermissionsForEveryone=1; } if ($PurgeLogFile !~ /[0-1]/) { $PurgeLogFile=0; } if ($ArchiveLogRecords !~ /[0-1]/) { $ArchiveLogRecords=0; } if ($KeepBackupOfHistoricFiles !~ /[0-1]/) { $KeepBackupOfHistoricFiles=0; } $DefaultFile[0]||='index.html'; if ($AuthenticatedUsersNotCaseSensitive !~ /[0-1]/) { $AuthenticatedUsersNotCaseSensitive=0; } if ($URLNotCaseSensitive !~ /[0-1]/) { $URLNotCaseSensitive=0; } if ($URLWithAnchor !~ /[0-1]/) { $URLWithAnchor=0; } $URLQuerySeparators =~ s/\s//g; if (! $URLQuerySeparators) { $URLQuerySeparators='?;'; } if ($URLWithQuery !~ /[0-1]/) { $URLWithQuery=0; } if ($URLReferrerWithQuery !~ /[0-1]/) { $URLReferrerWithQuery=0; } if ($WarningMessages !~ /[0-1]/) { $WarningMessages=1; } if ($DebugMessages !~ /[0-1]/) { $DebugMessages=1; } if ($NbOfLinesForCorruptedLog !~ /^\d+/ || $NbOfLinesForCorruptedLog<1) { $NbOfLinesForCorruptedLog=50; } if ($Expires !~ /^\d+/) { $Expires=0; } if ($DecodeUA !~ /[0-1]/) { $DecodeUA=0; } $MiscTrackerUrl||='/js/awstats_misc_tracker.js'; # Optional accuracy setup section if ($LevelForWormsDetection !~ /^\d+/) { $LevelForWormsDetection=0; } if ($LevelForRobotsDetection !~ /^\d+/) { $LevelForRobotsDetection=2; } if ($LevelForBrowsersDetection !~ /^\d+/) { $LevelForBrowsersDetection=2; } if ($LevelForOSDetection !~ /^\d+/) { $LevelForOSDetection=2; } if ($LevelForRefererAnalyze !~ /^\d+/) { $LevelForRefererAnalyze=2; } if ($LevelForFileTypesDetection !~ /^\d+/) { $LevelForFileTypesDetection=2; } if ($LevelForSearchEnginesDetection !~ /^\d+/) { $LevelForSearchEnginesDetection=2; } if ($LevelForKeywordsDetection !~ /^\d+/) { $LevelForKeywordsDetection=2; } # Optional extra setup section foreach my $extracpt (1..@ExtraName-1) { if ($ExtraStatTypes[$extracpt] !~ /[PHBL]/) { $ExtraStatTypes[$extracpt]='PHBL'; } if ($MaxNbOfExtra[$extracpt] !~ /^\d+$/ || $MaxNbOfExtra[$extracpt]<1) { $MaxNbOfExtra[$extracpt]=20; } if ($MinHitExtra[$extracpt] !~ /^\d+$/ || $MinHitExtra[$extracpt]<1) { $MinHitExtra[$extracpt]=1; } if (! $ExtraFirstColumnValues[$extracpt]) { error("Extra section number $extracpt is defined without ExtraSectionFirstColumnValues$extracpt parameter"); } if (! $ExtraFirstColumnFormat[$extracpt]) { $ExtraFirstColumnFormat[$extracpt] = '%s'; } } # Optional appearance setup section if ($MaxRowsInHTMLOutput !~ /^\d+/ || $MaxRowsInHTMLOutput<1) { $MaxRowsInHTMLOutput=1000; } if ($ShowMenu !~ /[01]/) { $ShowMenu=1; } if ($ShowMonthStats !~ /[01UVPHB]/) { $ShowMonthStats='UVPHB'; } if ($ShowDaysOfMonthStats !~ /[01VPHB]/) { $ShowDaysOfMonthStats='VPHB'; } if ($ShowDaysOfWeekStats !~ /[01PHBL]/) { $ShowDaysOfWeekStats='PHBL'; } if ($ShowHoursStats !~ /[01PHBL]/) { $ShowHoursStats='PHBL'; } if ($ShowDomainsStats !~ /[01PHB]/) { $ShowDomainsStats='PHB'; } if ($ShowHostsStats !~ /[01PHBL]/) { $ShowHostsStats='PHBL'; } if ($ShowAuthenticatedUsers !~ /[01PHBL]/) { $ShowAuthenticatedUsers=0; } if ($ShowRobotsStats !~ /[01HBL]/) { $ShowRobotsStats='HBL'; } if ($ShowWormsStats !~ /[01HBL]/) { $ShowWormsStats='HBL'; } if ($ShowEMailSenders !~ /[01HBML]/) { $ShowEMailSenders=0; } if ($ShowEMailReceivers !~ /[01HBML]/) { $ShowEMailReceivers=0; } if ($ShowSessionsStats !~ /[01]/) { $ShowSessionsStats=1; } if ($ShowPagesStats !~ /[01PBEX]/i) { $ShowPagesStats='PBEX'; } if ($ShowFileTypesStats !~ /[01HBC]/) { $ShowFileTypesStats='HB'; } if ($ShowFileSizesStats !~ /[01]/) { $ShowFileSizesStats=1; } if ($ShowOSStats !~ /[01]/) { $ShowOSStats=1; } if ($ShowBrowsersStats !~ /[01]/) { $ShowBrowsersStats=1; } if ($ShowScreenSizeStats !~ /[01]/) { $ShowScreenSizeStats=0; } if ($ShowOriginStats !~ /[01PH]/) { $ShowOriginStats='PH'; } if ($ShowKeyphrasesStats !~ /[01]/) { $ShowKeyphrasesStats=1; } if ($ShowKeywordsStats !~ /[01]/) { $ShowKeywordsStats=1; } if ($ShowClusterStats !~ /[01PHB]/) { $ShowClusterStats=0; } if ($ShowMiscStats !~ /[01anjdfrqwp]/) { $ShowMiscStats='a'; } if ($ShowHTTPErrorsStats !~ /[01]/) { $ShowHTTPErrorsStats=1; } if ($ShowSMTPErrorsStats !~ /[01]/) { $ShowSMTPErrorsStats=0; } if ($AddDataArrayMonthStats !~ /[01]/) { $AddDataArrayMonthStats=1; } if ($AddDataArrayShowDaysOfMonthStats !~ /[01]/) { $AddDataArrayShowDaysOfMonthStats=1; } if ($AddDataArrayShowDaysOfWeekStats !~ /[01]/) { $AddDataArrayShowDaysOfWeekStats=1; } if ($AddDataArrayShowHoursStats !~ /[01]/) { $AddDataArrayShowHoursStats=1; } my @maxnboflist=('Domain','HostsShown','LoginShown','RobotShown','WormsShown','PageShown','OsShown','BrowsersShown','ScreenSizesShown','RefererShown','KeyphrasesShown','KeywordsShown','EMailsShown'); my @maxnboflistdefaultval=(10,10,10,10,5,10,10,10,5,10,10,10,20); foreach my $i (0..(@maxnboflist-1)) { if (! $MaxNbOf{$maxnboflist[$i]} || $MaxNbOf{$maxnboflist[$i]} !~ /^\d+$/ || $MaxNbOf{$maxnboflist[$i]}<1) { $MaxNbOf{$maxnboflist[$i]}=$maxnboflistdefaultval[$i]; } } my @minhitlist=('Domain','Host','Login','Robot','Worm','File','Os','Browser','ScreenSize','Refer','Keyphrase','Keyword','EMail'); my @minhitlistdefaultval=(1,1,1,1,1,1,1,1,1,1,1,1,1); foreach my $i (0..(@minhitlist-1)) { if (! $MinHit{$minhitlist[$i]} || $MinHit{$minhitlist[$i]} !~ /^\d+$/ || $MinHit{$minhitlist[$i]}<1) { $MinHit{$minhitlist[$i]}=$minhitlistdefaultval[$i]; } } if ($FirstDayOfWeek !~ /[01]/) { $FirstDayOfWeek=1; } if ($UseFramesWhenCGI !~ /[01]/) { $UseFramesWhenCGI=1; } if ($DetailedReportsOnNewWindows !~ /[012]/) { $DetailedReportsOnNewWindows=1; } if ($ShowLinksOnUrl !~ /[01]/) { $ShowLinksOnUrl=1; } if ($MaxLengthOfShownURL !~ /^\d+/ || $MaxLengthOfShownURL<1) { $MaxLengthOfShownURL=64; } if ($ShowLinksToWhoIs !~ /[01]/) { $ShowLinksToWhoIs=0; } $Logo||='awstats_logo6.png'; $LogoLink||='http://awstats.sourceforge.net'; if ($BarWidth !~ /^\d+/ || $BarWidth<1) { $BarWidth=260; } if ($BarHeight !~ /^\d+/ || $BarHeight<1) { $BarHeight=90; } $color_Background =~ s/#//g; if ($color_Background !~ /^[0-9|A-H]+$/i) { $color_Background='FFFFFF'; } $color_TableBGTitle =~ s/#//g; if ($color_TableBGTitle !~ /^[0-9|A-H]+$/i) { $color_TableBGTitle='CCCCDD'; } $color_TableTitle =~ s/#//g; if ($color_TableTitle !~ /^[0-9|A-H]+$/i) { $color_TableTitle='000000'; } $color_TableBG =~ s/#//g; if ($color_TableBG !~ /^[0-9|A-H]+$/i) { $color_TableBG='CCCCDD'; } $color_TableRowTitle =~ s/#//g; if ($color_TableRowTitle !~ /^[0-9|A-H]+$/i) { $color_TableRowTitle='FFFFFF'; } $color_TableBGRowTitle =~ s/#//g; if ($color_TableBGRowTitle !~ /^[0-9|A-H]+$/i) { $color_TableBGRowTitle='ECECEC'; } $color_TableBorder =~ s/#//g; if ($color_TableBorder !~ /^[0-9|A-H]+$/i) { $color_TableBorder='ECECEC'; } $color_text =~ s/#//g; if ($color_text !~ /^[0-9|A-H]+$/i) { $color_text='000000'; } $color_textpercent =~ s/#//g; if ($color_textpercent !~ /^[0-9|A-H]+$/i) { $color_textpercent='606060'; } $color_titletext =~ s/#//g; if ($color_titletext !~ /^[0-9|A-H]+$/i) { $color_titletext='000000'; } $color_weekend =~ s/#//g; if ($color_weekend !~ /^[0-9|A-H]+$/i) { $color_weekend='EAEAEA'; } $color_link =~ s/#//g; if ($color_link !~ /^[0-9|A-H]+$/i) { $color_link='0011BB'; } $color_hover =~ s/#//g; if ($color_hover !~ /^[0-9|A-H]+$/i) { $color_hover='605040'; } $color_other =~ s/#//g; if ($color_other !~ /^[0-9|A-H]+$/i) { $color_other='666688'; } $color_u =~ s/#//g; if ($color_u !~ /^[0-9|A-H]+$/i) { $color_u='FFA060'; } $color_v =~ s/#//g; if ($color_v !~ /^[0-9|A-H]+$/i) { $color_v='F4F090'; } $color_p =~ s/#//g; if ($color_p !~ /^[0-9|A-H]+$/i) { $color_p='4477DD'; } $color_h =~ s/#//g; if ($color_h !~ /^[0-9|A-H]+$/i) { $color_h='66EEFF'; } $color_k =~ s/#//g; if ($color_k !~ /^[0-9|A-H]+$/i) { $color_k='2EA495'; } $color_s =~ s/#//g; if ($color_s !~ /^[0-9|A-H]+$/i) { $color_s='8888DD'; } $color_e =~ s/#//g; if ($color_e !~ /^[0-9|A-H]+$/i) { $color_e='CEC2E8'; } $color_x =~ s/#//g; if ($color_x !~ /^[0-9|A-H]+$/i) { $color_x='C1B2E2'; } # Correct param if default value is asked if ($ShowMonthStats eq '1') { $ShowMonthStats = 'UVPHB'; } if ($ShowDaysOfMonthStats eq '1') { $ShowDaysOfMonthStats = 'VPHB'; } if ($ShowDaysOfWeekStats eq '1') { $ShowDaysOfWeekStats = 'PHBL'; } if ($ShowHoursStats eq '1') { $ShowHoursStats = 'PHBL'; } if ($ShowDomainsStats eq '1') { $ShowDomainsStats = 'PHB'; } if ($ShowHostsStats eq '1') { $ShowHostsStats = 'PHBL'; } if ($ShowEMailSenders eq '1') { $ShowEMailSenders = 'HBML'; } if ($ShowEMailReceivers eq '1') { $ShowEMailReceivers = 'HBML'; } if ($ShowAuthenticatedUsers eq '1') { $ShowAuthenticatedUsers = 'PHBL'; } if ($ShowRobotsStats eq '1') { $ShowRobotsStats = 'HBL'; } if ($ShowWormsStats eq '1') { $ShowWormsStats = 'HBL'; } if ($ShowPagesStats eq '1') { $ShowPagesStats = 'PBEX'; } if ($ShowFileTypesStats eq '1') { $ShowFileTypesStats = 'HB'; } if ($ShowOriginStats eq '1') { $ShowOriginStats = 'PH'; } if ($ShowClusterStats eq '1') { $ShowClusterStats = 'PHB'; } if ($ShowMiscStats eq '1') { $ShowMiscStats = 'anjdfrqwp'; } # Convert extra sections data into @ExtraConditionType, @ExtraConditionTypeVal... foreach my $extranum (1..@ExtraName-1) { my $part=0; foreach my $conditioncouple (split(/\s*\|\|\s*/, $ExtraCondition[$extranum])) { my ($conditiontype, $conditiontypeval)=split(/[,:]/,$conditioncouple,2); $ExtraConditionType[$extranum][$part]=$conditiontype; if ($conditiontypeval =~ /^REGEX\[(.*)\]$/i) { $conditiontypeval=$1; } #else { $conditiontypeval=quotemeta($conditiontypeval); } $ExtraConditionTypeVal[$extranum][$part]=qr/$conditiontypeval/i; $part++; } $part=0; foreach my $rowkeycouple (split(/\s*\|\|\s*/, $ExtraFirstColumnValues[$extranum])) { my ($rowkeytype, $rowkeytypeval)=split(/[,:]/,$rowkeycouple,2); $ExtraFirstColumnValuesType[$extranum][$part]=$rowkeytype; if ($rowkeytypeval =~ /^REGEX\[(.*)\]$/i) { $rowkeytypeval=$1; } #else { $rowkeytypeval=quotemeta($rowkeytypeval); } $ExtraFirstColumnValuesTypeVal[$extranum][$part]=qr/$rowkeytypeval/i; $part++; } } # Show definitive value for major parameters if ($Debug) { debug(" LogFile='$LogFile'",2); debug(" LogFormat='$LogFormat'",2); debug(" LogSeparator='$LogSeparator'",2); debug(" DNSLookup='$DNSLookup'",2); debug(" DirData='$DirData'",2); debug(" DirCgi='$DirCgi'",2); debug(" DirIcons='$DirIcons'",2); debug(" SiteDomain='$SiteDomain'",2); debug(" MiscTrackerUrl='$MiscTrackerUrl'",2); foreach (keys %MaxNbOf) { debug(" MaxNbOf{$_}=$MaxNbOf{$_}",2); } foreach (keys %MinHit) { debug(" MinHit{$_}=$MinHit{$_}",2); } foreach my $extranum (1..@ExtraName-1) { debug(" ExtraCodeFilter[$extranum] is array ".join(',',@{$ExtraCodeFilter[$extranum]}),2); debug(" ExtraConditionType[$extranum] is array ".join(',',@{$ExtraConditionType[$extranum]}),2); debug(" ExtraConditionTypeVal[$extranum] is array ".join(',',@{$ExtraConditionTypeVal[$extranum]}),2); debug(" ExtraFirstColumnValuesType[$extranum] is array ".join(',',@{$ExtraFirstColumnValuesType[$extranum]}),2); debug(" ExtraFirstColumnValuesTypeVal[$extranum] is array ".join(',',@{$ExtraFirstColumnValuesTypeVal[$extranum]}),2); } } # Deny URLWithQueryWithOnlyFollowingParameters and URLWithQueryWithoutFollowingParameters both set if (@URLWithQueryWithOnly && @URLWithQueryWithout) { error("URLWithQueryWithOnlyFollowingParameters and URLWithQueryWithoutFollowingParameters can't be both set at the same time"); } # Deny $ShowHTTPErrorsStats and $ShowSMTPErrorsStats both set if ($ShowHTTPErrorsStats && $ShowSMTPErrorsStats) { error("ShowHTTPErrorsStats and ShowSMTPErrorsStats can't be both set at the same time"); } # Deny LogFile if contains a pipe and PurgeLogFile || ArchiveLogRecords set on if (($PurgeLogFile || $ArchiveLogRecords) && $LogFile =~ /\|\s*$/) { error("A pipe in log file name is not allowed if PurgeLogFile and ArchiveLogRecords are not set to 0"); } # If not a migrate, check if DirData is OK if (! $MigrateStats && ! -d $DirData) { if ($CreateDirDataIfNotExists) { if ($Debug) { debug(" Make directory $DirData",2); } my $mkdirok=mkdir "$DirData", 0766; if (! $mkdirok) { error("$PROG failed to create directory DirData (DirData=\"$DirData\", CreateDirDataIfNotExists=$CreateDirDataIfNotExists)."); } } else { error("AWStats database directory defined in config file by 'DirData' parameter ($DirData) does not exist or is not writable."); } } if ($LogType eq 'S') { $NOTSORTEDRECORDTOLERANCE=1000000; } } #------------------------------------------------------------------------------ # Function: Common function used by init function of plugins # Parameters: AWStats version required by plugin # Input: $VERSION # Output: None # Return: '' if ok, "Error: xxx" if error #------------------------------------------------------------------------------ sub Check_Plugin_Version { my $PluginNeedAWStatsVersion=shift; if (! $PluginNeedAWStatsVersion) { return 0; } $VERSION =~ /^(\d+)\.(\d+)/; my $numAWStatsVersion=($1*1000)+$2; $PluginNeedAWStatsVersion =~ /^(\d+)\.(\d+)/; my $numPluginNeedAWStatsVersion=($1*1000)+$2; if ($numPluginNeedAWStatsVersion > $numAWStatsVersion) { return "Error: AWStats version $PluginNeedAWStatsVersion or higher is required. Detected $VERSION."; } return ''; } #------------------------------------------------------------------------------ # Function: Return a checksum for an array of string # Parameters: Array of string # Input: None # Output: None # Return: Checksum number #------------------------------------------------------------------------------ sub CheckSum { my $string=shift; my $checksum=0; # use MD5; # $checksum = MD5->hexhash($string); my $i=0; my $j=0; while ($i < length($string)) { my $c=substr($string,$i,1); $checksum+=(ord($c)<<(8*$j)); if ($j++ > 3) { $j=0; } $i++; } return $checksum; } #------------------------------------------------------------------------------ # Function: Load plugins files # Parameters: None # Input: $DIR @PluginsToLoad # Output: None # Return: None #------------------------------------------------------------------------------ sub Read_Plugins { # Check plugin files in common possible directories : # Windows and standard package: "$DIR/plugins" (plugins in same dir than awstats.pl) # Redhat : "/usr/local/awstats/wwwroot/cgi-bin/plugins" # Debian package : "/usr/share/awstats/plugins" my @PossiblePluginsDir=("$DIR/plugins","/usr/local/awstats/wwwroot/cgi-bin/plugins","/usr/share/awstats/plugins"); my %DirAddedInINC=(); foreach my $key (keys %NoLoadPlugin) { if ($NoLoadPlugin{$key} < 0) { push @PluginsToLoad, $key; } } if ($Debug) { debug("Call to Read_Plugins with list: ".join(',',@PluginsToLoad)); } foreach my $plugininfo (@PluginsToLoad) { my ($pluginfile,$pluginparam)=split(/\s+/,$plugininfo,2); $pluginfile =~ s/\.pm$//i; $pluginfile =~ /([^\/\\]+)$/; my $pluginname=$1; # pluginname is pluginfile without any path # Check if plugin is not disabled if ($NoLoadPlugin{$pluginname} && $NoLoadPlugin{$pluginname} > 0) { if ($Debug) { debug(" Plugin load for '$pluginfile' has been disabled from parameters"); } next; } if ($pluginname) { if (! $PluginsLoaded{'init'}{"$pluginname"}) { # Plugin not already loaded my %pluginisfor=('timehires'=>'u','ipv6'=>'u','hashfiles'=>'u','geoip'=>'u','geoipfree'=>'u', 'geoip_region_maxmind'=>'mou','timezone'=>'ou', 'decodeutfkeys'=>'o','hostinfo'=>'o','rawlog'=>'o','userinfo'=>'o','urlalias'=>'o','tooltips'=>'o'); if ($pluginisfor{$pluginname}) { # If it's a known plugin, may be we don't need to load it # Do not load "menu handler plugins" if output only and mainleft frame if (! $UpdateStats && scalar keys %HTMLOutput && $FrameName eq 'mainleft' && $pluginisfor{$pluginname} !~ /m/) { $PluginsLoaded{'init'}{"$pluginname"}=1; next; } # Do not load "update plugins" if output only if (! $UpdateStats && scalar keys %HTMLOutput && $pluginisfor{$pluginname} !~ /o/) { $PluginsLoaded{'init'}{"$pluginname"}=1; next; } # Do not load "output plugins" if update only if ($UpdateStats && ! scalar keys %HTMLOutput && $pluginisfor{$pluginname} !~ /u/) { $PluginsLoaded{'init'}{"$pluginname"}=1; next; } } # Load plugin foreach my $dir (@PossiblePluginsDir) { my $searchdir=$dir; if ($searchdir && (!($searchdir =~ /\/$/)) && (!($searchdir =~ /\\$/)) ) { $searchdir .= "/"; } my $pluginpath="${searchdir}${pluginfile}.pm"; if (-s "$pluginpath") { $PluginDir="${searchdir}"; # Set plugin dir if ($Debug) { debug(" Try to init plugin '$pluginname' ($pluginpath) with param '$pluginparam'",1); } if (! $DirAddedInINC{"$dir"}) { push @INC, "$dir"; $DirAddedInINC{"$dir"}=1; } my $loadret=0; my $modperl=$ENV{"MOD_PERL"}? eval { require mod_perl; $mod_perl::VERSION >= 1.99 ? 2 : 1 } : 0; if ($modperl == 2) { $loadret=require "$pluginpath"; } else { $loadret=require "$pluginfile.pm"; } if (! $loadret || $loadret =~ /^error/i) { # Load failed, we stop here error("Plugin load for plugin '$pluginname' failed with return code: $loadret"); } my $ret; # To get init return my $initfunction="\$ret=Init_$pluginname('$pluginparam')"; my $initret=eval("$initfunction"); if ($initret eq 'xxx') { $initret='Error: The PluginHooksFunctions variable defined in plugin file does not contain list of hooked functions'; } if (! $initret || $initret =~ /^error/i) { # Init function failed, we stop here error("Plugin init for plugin '$pluginname' failed with return code: ".($initret?"$initret":"$@ (A module required by plugin might be missing).")); } # Plugin load and init successfull foreach my $elem (split(/\s+/,$initret)) { # Some functions can only be plugged once my @uniquefunc=('GetCountryCodeByName','GetCountryCodeByAddr','ChangeTime','GetTimeZoneTitle','GetTime','SearchFile','LoadCache','SaveHash','ShowMenu'); my $isuniquefunc=0; foreach my $function (@uniquefunc) { if ("$elem" eq "$function") { # We try to load a 'unique' function, so we check and stop if already loaded foreach my $otherpluginname (keys %{$PluginsLoaded{"$elem"}}) { error("Conflict between plugin '$pluginname' and '$otherpluginname'. They both implements the 'must be unique' function '$elem'.\nYou must choose between one of them. Using together is not possible."); } $isuniquefunc=1; last; } } if ($isuniquefunc) { # TODO Use $PluginsLoaded{"$elem"}="$pluginname"; for unique func $PluginsLoaded{"$elem"}{"$pluginname"}=1; } else { $PluginsLoaded{"$elem"}{"$pluginname"}=1; } if ("$elem" =~ /SectionInitHashArray/) { $AtLeastOneSectionPlugin=1; } } $PluginsLoaded{'init'}{"$pluginname"}=1; if ($Debug) { debug(" Plugin '$pluginname' now hooks functions '$initret'",1); } last; } } if (! $PluginsLoaded{'init'}{"$pluginname"}) { error("AWStats config file contains a directive to load plugin \"$pluginname\" (LoadPlugin=\"$plugininfo\") but AWStats can't open plugin file \"$pluginfile.pm\" for read.\nCheck if file is in \"".($PossiblePluginsDir[0])."\" directory and is readable."); } } else { warning("Warning: Tried to load plugin \"$pluginname\" twice. Fix config file."); } } else { error("Plugin \"$pluginfile\" is not a valid plugin name."); } } # In output mode, geo ip plugins are not loaded, so message changes are done here (can't be done in plugin init function) if ($PluginsLoaded{'init'}{'geoip'} || $PluginsLoaded{'init'}{'geoipfree'}) { $Message[17]=$Message[25]=$Message[148]; } } #------------------------------------------------------------------------------ # Function: Read history file and create/update tmp history file # Parameters: year,month,withupdate,withpurge,part_to_load[,lastlinenb,lastlineoffset,lastlinechecksum] # Input: $DirData $PROG $FileSuffix $LastLine # Output: None # Return: Tmp history file name created/updated or '' if withupdate is 0 #------------------------------------------------------------------------------ sub Read_History_With_TmpUpdate { my $year=sprintf("%04i",shift||0); my $month=sprintf("%02i",shift||0); my $withupdate=shift||0; my $withpurge=shift||0; my $part=shift||''; my $xml=($BuildHistoryFormat eq 'xml'?1:0); my $xmleb=''; my $xmlrb=''; my $lastlinenb=shift||0; my $lastlineoffset=shift||0; my $lastlinechecksum=shift||0; my %allsections=('general'=>1,'misc'=>2,'time'=>3,'visitor'=>4,'day'=>5, 'domain'=>6,'cluster'=>7,'login'=>8,'robot'=>9,'worms'=>10,'emailsender'=>11,'emailreceiver'=>12, 'session'=>13,'sider'=>14,'filetypes'=>15, 'os'=>16,'browser'=>17,'screensize'=>18,'unknownreferer'=>19,'unknownrefererbrowser'=>20, 'origin'=>21,'sereferrals'=>22,'pagerefs'=>23, 'searchwords'=>24,'keywords'=>25, 'errors'=>26); my $order=(scalar keys %allsections)+1; foreach (keys %TrapInfosForHTTPErrorCodes) { $allsections{"sider_$_"}=$order++; } foreach (1..@ExtraName-1) { $allsections{"extra_$_"}=$order++; } foreach (keys %{$PluginsLoaded{'SectionInitHashArray'}}) { $allsections{"plugin_$_"}=$order++; } my $withread=0; # Variable used to read old format history files my $readvisitorforbackward=0; # In standard use of AWStats, the DayRequired variable is always empty if ($DayRequired) { if ($Debug) { debug("Call to Read_History_With_TmpUpdate [$year,$month,withupdate=$withupdate,withpurge=$withpurge,part=$part,lastlinenb=$lastlinenb,lastlineoffset=$lastlineoffset,lastlinechecksum=$lastlinechecksum] ($DayRequired)"); } } else { if ($Debug) { debug("Call to Read_History_With_TmpUpdate [$year,$month,withupdate=$withupdate,withpurge=$withpurge,part=$part,lastlinenb=$lastlinenb,lastlineoffset=$lastlineoffset,lastlinechecksum=$lastlinechecksum]"); } } # Define SectionsToLoad (which sections to load) my %SectionsToLoad = (); if ($part eq 'all') { # Load all needed sections my $order=1; $SectionsToLoad{'general'}=$order++; # When $SectionsToLoad{'time'}=$order++; # Always loaded because needed to count TotalPages, TotalHits, TotalBandwidth if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowHostsStats) || $HTMLOutput{'allhosts'} || $HTMLOutput{'lasthosts'} || $HTMLOutput{'unknownip'}) { $SectionsToLoad{'visitor'}=$order++; } # Must be before day, sider and session section if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && ($ShowDaysOfWeekStats || $ShowDaysOfMonthStats)) || $HTMLOutput{'alldays'}) { $SectionsToLoad{'day'}=$order++; } # Who if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowDomainsStats) || $HTMLOutput{'alldomains'}) { $SectionsToLoad{'domain'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowAuthenticatedUsers) || $HTMLOutput{'alllogins'} || $HTMLOutput{'lastlogins'}) { $SectionsToLoad{'login'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowRobotsStats) || $HTMLOutput{'allrobots'} || $HTMLOutput{'lastrobots'}) { $SectionsToLoad{'robot'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowWormsStats) || $HTMLOutput{'allworms'} || $HTMLOutput{'lastworms'}) { $SectionsToLoad{'worms'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowEMailSenders) || $HTMLOutput{'allemails'} || $HTMLOutput{'lastemails'}) { $SectionsToLoad{'emailsender'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowEMailReceivers) || $HTMLOutput{'allemailr'} || $HTMLOutput{'lastemailr'}) { $SectionsToLoad{'emailreceiver'}=$order++; } # Navigation if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowSessionsStats) || $HTMLOutput{'sessions'}) { $SectionsToLoad{'session'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowPagesStats) || $HTMLOutput{'urldetail'} || $HTMLOutput{'urlentry'} || $HTMLOutput{'urlexit'}) { $SectionsToLoad{'sider'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowFileTypesStats) || $HTMLOutput{'filetypes'}) { $SectionsToLoad{'filetypes'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowOSStats) || $HTMLOutput{'osdetail'}) { $SectionsToLoad{'os'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowBrowsersStats) || $HTMLOutput{'browserdetail'}) { $SectionsToLoad{'browser'}=$order++; } if ($UpdateStats || $MigrateStats || $HTMLOutput{'unknownos'}) { $SectionsToLoad{'unknownreferer'}=$order++; } if ($UpdateStats || $MigrateStats || $HTMLOutput{'unknownbrowser'}) { $SectionsToLoad{'unknownrefererbrowser'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowScreenSizeStats)) { $SectionsToLoad{'screensize'}=$order++; } # Referers if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowOriginStats) || $HTMLOutput{'origin'}) { $SectionsToLoad{'origin'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowOriginStats) || $HTMLOutput{'refererse'}) { $SectionsToLoad{'sereferrals'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowOriginStats) || $HTMLOutput{'refererpages'}) { $SectionsToLoad{'pagerefs'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowKeyphrasesStats) || $HTMLOutput{'keyphrases'} || $HTMLOutput{'keywords'}) { $SectionsToLoad{'searchwords'}=$order++; } if (! $withupdate && $HTMLOutput{'main'} && $ShowKeywordsStats) { $SectionsToLoad{'keywords'}=$order++; } # If we update, dont need to load # Others if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowMiscStats)) { $SectionsToLoad{'misc'}=$order++; } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && ($ShowHTTPErrorsStats || $ShowSMTPErrorsStats)) || $HTMLOutput{'errors'}) { $SectionsToLoad{'errors'}=$order++; } foreach (keys %TrapInfosForHTTPErrorCodes) { if ($UpdateStats || $MigrateStats || $HTMLOutput{"errors$_"}) { $SectionsToLoad{"sider_$_"}=$order++; } } if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ShowClusterStats)) { $SectionsToLoad{'cluster'}=$order++; } foreach (1..@ExtraName-1) { if ($UpdateStats || $MigrateStats || ($HTMLOutput{'main'} && $ExtraStatTypes[$_]) || $HTMLOutput{"extra$_"}) { $SectionsToLoad{"extra_$_"}=$order++; } } foreach (keys %{$PluginsLoaded{'SectionInitHashArray'}}) { if ($UpdateStats || $MigrateStats || $HTMLOutput{"plugin_$_"}) { $SectionsToLoad{"plugin_$_"}=$order++; } } } else { # Load only required sections my $order=1; foreach (split(/\s+/,$part)) { $SectionsToLoad{$_}=$order++; } } # Define SectionsToSave (which sections to save) my %SectionsToSave = (); if ($withupdate) { %SectionsToSave=%allsections; } if ($Debug) { debug(" List of sections marked for load : ".join(' ',(sort { $SectionsToLoad{$a} <=> $SectionsToLoad{$b} } keys %SectionsToLoad)),2); debug(" List of sections marked for save : ".join(' ',(sort { $SectionsToSave{$a} <=> $SectionsToSave{$b} } keys %SectionsToSave)),2); } # Define value for filetowrite and filetoread (Month before Year kept for backward compatibility) my $filetowrite=''; my $filetoread=''; if ($HistoryAlreadyFlushed{"$year$month"} && -s "$DirData/$PROG$month$year$FileSuffix.tmp.$$") { # tmp history file was already flushed $filetoread="$DirData/$PROG$month$year$FileSuffix.tmp.$$"; $filetowrite="$DirData/$PROG$month$year$FileSuffix.tmp.$$.bis"; } else { $filetoread="$DirData/$PROG$DayRequired$month$year$FileSuffix.txt"; $filetowrite="$DirData/$PROG$month$year$FileSuffix.tmp.$$"; } if ($Debug) { debug(" History file to read is '$filetoread'",2); } # Is there an old data file to read or, if migrate, can we open the file for read if (-s $filetoread || $MigrateStats) { $withread=1; } # Open files if ($withread) { open(HISTORY,$filetoread) || error("Couldn't open file \"$filetoread\" for read: $!","","",$MigrateStats); binmode HISTORY; # Avoid premature EOF due to history files corrupted with \cZ or bin chars } if ($withupdate) { open(HISTORYTMP,">$filetowrite") || error("Couldn't open file \"$filetowrite\" for write: $!"); binmode HISTORYTMP; if ($xml) { print HISTORYTMP "\n\n"; } Save_History("header",$year,$month); } # Loop on read file my $xmlold=0; if ($withread) { my $countlines=0; my $versionnum=0; my @field=(); while () { chomp $_; s/\r//; $countlines++; # Test if it's xml if (! $xmlold && $_ =~ /^ int($field[1])) { $FirstTime{$year.$month}=int($field[1]);}; next; } if ($field[0] eq 'LastTime' || $field[0] eq "${xmlrb}LastTime") { if (! $LastTime{$year.$month} || $LastTime{$year.$month} < int($field[1])) { $LastTime{$year.$month}=int($field[1]);}; next; } if ($field[0] eq 'LastUpdate' || $field[0] eq "${xmlrb}LastUpdate") { if ($LastUpdate < $field[1]) { $LastUpdate=int($field[1]); #$LastUpdateLinesRead=int($field[2]); #$LastUpdateNewLinesRead=int($field[3]); #$LastUpdateLinesCorrupted=int($field[4]); }; next; } if ($field[0] eq 'TotalVisits' || $field[0] eq "${xmlrb}TotalVisits") { if (! $withupdate) { $MonthVisits{$year.$month}+=int($field[1]); } # Save in MonthVisits also if migrate from a file < 4.x for backward compatibility if ($MigrateStats && $versionnum < 4000 && ! $MonthVisits{$year.$month}) { if ($Debug) { debug("File is version < 4000. We save ".int($field[1])." visits in DayXxx arrays",1); } $DayHits{$year.$month."00"}+=0; $DayVisits{$year.$month."00"}+=int($field[1]); } next; } if ($field[0] eq 'TotalUnique' || $field[0] eq "${xmlrb}TotalUnique") { if (! $withupdate) { $MonthUnique{$year.$month}+=int($field[1]); } next; } if ($field[0] eq 'MonthHostsKnown' || $field[0] eq "${xmlrb}MonthHostsKnown") { if (! $withupdate) { $MonthHostsKnown{$year.$month}+=int($field[1]); } next; } if ($field[0] eq 'MonthHostsUnknown' || $field[0] eq "${xmlrb}MonthHostsUnknown") { if (! $withupdate) { $MonthHostsUnknown{$year.$month}+=int($field[1]); } next; } if (($field[0] eq 'END_GENERAL' || $field[0] eq "${xmleb}END_GENERAL") # END_GENERAL didn't exist for history files < 5.0 || ($versionnum < 5000 && $SectionsToLoad{"general"} && $FirstTime{$year.$month} && $LastTime{$year.$month}) ) { if ($Debug) { debug(" End of GENERAL section"); } # Show migrate warning for backward compatibility if ($versionnum < 5000 && ! $MigrateStats && ! $BadFormatWarning{$year.$month}) { if ($FrameName ne 'mainleft') { $BadFormatWarning{$year.$month}=1; my $message="Warning: Data file '$filetoread' has an old history file format (version $versionnum). You should upgrade it...\nFrom command line: $PROG.$Extension -migrate=\"$filetoread\""; if ($ENV{'GATEWAY_INTERFACE'} && $AllowToUpdateStatsFromBrowser) { $message.="\nFrom your browser with URL: http://".$ENV{"SERVER_NAME"}.$ENV{"SCRIPT_NAME"}."?migrate=$filetoread"; } warning("$message"); } } if (! ($versionnum < 5000) && $MigrateStats && ! $BadFormatWarning{$year.$month}) { $BadFormatWarning{$year.$month}=1; warning("Warning: You are migrating a file that is already a recent version (migrate not required for files version $versionnum).","","",1); } # If migrate and version < 4.x we need to include BEGIN_UNKNOWNIP into BEGIN_VISITOR for backward compatibility if ($MigrateStats && $versionnum < 4000) { if ($Debug) { debug("File is version < 4000. We add UNKNOWNIP in sections to load",1); } $SectionsToLoad{'unknownip'}=99; } delete $SectionsToLoad{'general'}; if ($SectionsToSave{'general'}) { Save_History('general',$year,$month,$lastlinenb,$lastlineoffset,$lastlinechecksum); delete $SectionsToSave{'general'}; } # Test for backward compatibility if ($versionnum < 5000 && ! $withupdate) { # We must find another way to init MonthUnique MonthHostsKnown and MonthHostsUnknown if ($Debug) { debug(" We ask to count MonthUnique, MonthHostsKnown and MonthHostsUnknown in visitor section because they are not stored in general section for this data file (version $versionnum)."); } $readvisitorforbackward=($SectionsToLoad{"visitor"}?1:2); $SectionsToLoad{"visitor"}=4; } else { if (! scalar %SectionsToLoad) { if ($Debug) { debug(" Stop reading history file. Got all we need."); } last; } } if ($versionnum >= 5000) { next; } # We can forget 'END_GENERAL' line and read next one } # BEGIN_MISC if ($field[0] eq 'BEGIN_MISC') { if ($Debug) { debug(" Begin of MISC section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'misc'}) { $countloaded++; if ($field[1]) { $_misc_p{$field[0]}+=int($field[1]); } if ($field[2]) { $_misc_h{$field[0]}+=int($field[2]); } if ($field[3]) { $_misc_k{$field[0]}+=int($field[3]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_MISC' || $field[0] eq "${xmleb}END_MISC" || ! $_); if ($field[0] ne 'END_MISC' && $field[0] ne "${xmleb}END_MISC") { error("History file \"$filetoread\" is corrupted (End of section MISC not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of MISC section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'misc'}; if ($SectionsToSave{'misc'}) { Save_History('misc',$year,$month); delete $SectionsToSave{'misc'}; if ($withpurge) { %_misc_p=(); %_misc_h=(); %_misc_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_CLUSTER if ($field[0] eq 'BEGIN_CLUSTER') { if ($Debug) { debug(" Begin of CLUSTER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'cluster'}) { $countloaded++; if ($field[1]) { $_cluster_p{$field[0]}+=int($field[1]); } if ($field[2]) { $_cluster_h{$field[0]}+=int($field[2]); } if ($field[3]) { $_cluster_k{$field[0]}+=int($field[3]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_CLUSTER' || $field[0] eq "${xmleb}END_CLUSTER" || ! $_); if ($field[0] ne 'END_CLUSTER' && $field[0] ne "${xmleb}END_CLUSTER") { error("History file \"$filetoread\" is corrupted (End of section CLUSTER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of CLUSTER section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'cluster'}; if ($SectionsToSave{'cluster'}) { Save_History('cluster',$year,$month); delete $SectionsToSave{'cluster'}; if ($withpurge) { %_cluster_p=(); %_cluster_h=(); %_cluster_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_TIME if ($field[0] eq 'BEGIN_TIME') { my $monthpages=0;my $monthhits=0;my $monthbytes=0; my $monthnotviewedpages=0;my $monthnotviewedhits=0;my $monthnotviewedbytes=0; if ($Debug) { debug(" Begin of TIME section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0] ne '') { # Test on ne '' because field[0] is '0' for hour 0) $count++; if ($SectionsToLoad{'time'}) { if ($withupdate || $MonthRequired eq 'all' || $MonthRequired eq "$month") { # Still required $countloaded++; if ($field[1]) { $_time_p[$field[0]]+=int($field[1]); } if ($field[2]) { $_time_h[$field[0]]+=int($field[2]); } if ($field[3]) { $_time_k[$field[0]]+=int($field[3]); } if ($field[4]) { $_time_nv_p[$field[0]]+=int($field[4]); } if ($field[5]) { $_time_nv_h[$field[0]]+=int($field[5]); } if ($field[6]) { $_time_nv_k[$field[0]]+=int($field[6]); } } $monthpages+=int($field[1]); $monthhits+=int($field[2]); $monthbytes+=int($field[3]); $monthnotviewedpages+=int($field[4]||0); $monthnotviewedhits+=int($field[5]||0); $monthnotviewedbytes+=int($field[6]||0); } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_TIME' || $field[0] eq "${xmleb}END_TIME" || ! $_); if ($field[0] ne 'END_TIME' && $field[0] ne "${xmleb}END_TIME") { error("History file \"$filetoread\" is corrupted (End of section TIME not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of TIME section ($count entries, $countloaded loaded)"); } $MonthPages{$year.$month}+=$monthpages; $MonthHits{$year.$month}+=$monthhits; $MonthBytes{$year.$month}+=$monthbytes; $MonthNotViewedPages{$year.$month}+=$monthnotviewedpages; $MonthNotViewedHits{$year.$month}+=$monthnotviewedhits; $MonthNotViewedBytes{$year.$month}+=$monthnotviewedbytes; delete $SectionsToLoad{'time'}; if ($SectionsToSave{'time'}) { Save_History('time',$year,$month); delete $SectionsToSave{'time'}; if ($withpurge) { @_time_p=(); @_time_h=(); @_time_k=(); @_time_nv_p=(); @_time_nv_h=(); @_time_nv_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_ORIGIN if ($field[0] eq 'BEGIN_ORIGIN') { if ($Debug) { debug(" Begin of ORIGIN section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'origin'}) { if ($field[0] eq 'From0') { $_from_p[0]+=$field[1]; $_from_h[0]+=$field[2]; } elsif ($field[0] eq 'From1') { $_from_p[1]+=$field[1]; $_from_h[1]+=$field[2]; } elsif ($field[0] eq 'From2') { $_from_p[2]+=$field[1]; $_from_h[2]+=$field[2]; } elsif ($field[0] eq 'From3') { $_from_p[3]+=$field[1]; $_from_h[3]+=$field[2]; } elsif ($field[0] eq 'From4') { $_from_p[4]+=$field[1]; $_from_h[4]+=$field[2]; } elsif ($field[0] eq 'From5') { $_from_p[5]+=$field[1]; $_from_h[5]+=$field[2]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_ORIGIN' || $field[0] eq "${xmleb}END_ORIGIN" || ! $_); if ($field[0] ne 'END_ORIGIN' && $field[0] ne "${xmleb}END_ORIGIN") { error("History file \"$filetoread\" is corrupted (End of section ORIGIN not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of ORIGIN section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'origin'}; if ($SectionsToSave{'origin'}) { Save_History('origin',$year,$month); delete $SectionsToSave{'origin'}; if ($withpurge) { @_from_p=(); @_from_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_DAY if ($field[0] eq 'BEGIN_DAY') { if ($Debug) { debug(" Begin of DAY section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'day'}) { $countloaded++; if ($field[1]) { $DayPages{$field[0]}+=int($field[1]); } $DayHits{$field[0]}+=int($field[2]); # DayHits always load (should be >0 and if not it's a day YYYYMM00 resulting of an old file migration) if ($field[3]) { $DayBytes{$field[0]}+=int($field[3]); } if ($field[4]) { $DayVisits{$field[0]}+=int($field[4]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_DAY' || $field[0] eq "${xmleb}END_DAY" || ! $_); if ($field[0] ne 'END_DAY' && $field[0] ne "${xmleb}END_DAY") { error("History file \"$filetoread\" is corrupted (End of section DAY not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of DAY section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'day'}; # WE DO NOT SAVE SECTION NOW BECAUSE VALUES CAN BE CHANGED AFTER READING VISITOR #if ($SectionsToSave{'day'}) { # Must be made after read of visitor # Save_History('day',$year,$month); delete $SectionsToSave{'day'}; # if ($withpurge) { %DayPages=(); %DayHits=(); %DayBytes=(); %DayVisits=(); } #} if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_VISITOR if ($field[0] eq 'BEGIN_VISITOR') { if ($Debug) { debug(" Begin of VISITOR section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; # For backward compatibility if ($readvisitorforbackward) { if ($field[1]) { $MonthUnique{$year.$month}++; } if ($MonthRequired ne 'all') { if ($field[0] !~ /^\d+\.\d+\.\d+\.\d+$/ && $field[0] !~ /^[0-9A-F]*:/i) { $MonthHostsKnown{$year.$month}++; } else { $MonthHostsUnknown{$year.$month}++; } } } # Process data saved in 'wait' arrays if ($withupdate && $_waithost_e{$field[0]}){ my $timehostl=int($field[4]||0); my $timehosts=int($field[5]||0); my $newtimehosts=($_waithost_s{$field[0]}?$_waithost_s{$field[0]}:$_host_s{$field[0]}); my $newtimehostl=($_waithost_l{$field[0]}?$_waithost_l{$field[0]}:$_host_l{$field[0]}); if ($newtimehosts > $timehostl + $VISITTIMEOUT ) { if ($Debug) { debug(" Visit for $field[0] in 'wait' arrays is a new visit different than last in history",4); } if ($field[6]) { $_url_x{$field[6]}++; } $_url_e{$_waithost_e{$field[0]}}++; $newtimehosts =~ /^(\d\d\d\d\d\d\d\d)/; $DayVisits{$1}++; if ($timehosts && $timehostl) { $_session{GetSessionRange($timehosts,$timehostl)}++; } if ($_waithost_s{$field[0]}) { # First session found in log was followed by another one so it's finished $_session{GetSessionRange($newtimehosts,$newtimehostl)}++; } # Here $_host_l $_host_s and $_host_u are correctly defined } else { if ($Debug) { debug(" Visit for $field[0] in 'wait' arrays is following of last visit in history",4); } if ($_waithost_s{$field[0]}) { # First session found in log was followed by another one so it's finished $_session{GetSessionRange(MinimumButNoZero($timehosts,$newtimehosts),$timehostl>$newtimehostl?$timehostl:$newtimehostl)}++; # Here $_host_l $_host_s and $_host_u are correctly defined } else { # We correct $_host_l $_host_s and $_host_u if ($timehostl > $newtimehostl) { $_host_l{$field[0]}=$timehostl; $_host_u{$field[0]}=$field[6]; } if ($timehosts < $newtimehosts) { $_host_s{$field[0]}=$timehosts; } } } delete $_waithost_e{$field[0]}; delete $_waithost_l{$field[0]}; delete $_waithost_s{$field[0]}; delete $_waithost_u{$field[0]}; } # Load records if ($readvisitorforbackward!=2 && $SectionsToLoad{'visitor'}) { # if readvisitorforbackward==2 we do not load my $loadrecord=0; if ($withupdate) { $loadrecord=1; } else { if ($HTMLOutput{'allhosts'} || $HTMLOutput{'lasthosts'}) { if ((!$FilterIn{'host'} || $field[0] =~ /$FilterIn{'host'}/i) && (!$FilterEx{'host'} || $field[0] !~ /$FilterEx{'host'}/i)) { $loadrecord=1; } } elsif ($MonthRequired eq 'all' || $field[2] >= $MinHit{'Host'}) { if ($HTMLOutput{'unknownip'} && ($field[0] =~ /^\d+\.\d+\.\d+\.\d+$/ || $field[0] =~ /^[0-9A-F]*:/i)) { $loadrecord=1; } elsif ($HTMLOutput{'main'} && ($MonthRequired eq 'all' || $countloaded < $MaxNbOf{'HostsShown'})) { $loadrecord=1; } } } if ($loadrecord) { if ($field[1]) { $_host_p{$field[0]}+=$field[1]; } if ($field[2]) { $_host_h{$field[0]}+=$field[2]; } if ($field[3]) { $_host_k{$field[0]}+=$field[3]; } if ($field[4] && ! $_host_l{$field[0]}) { # We save last connexion params if not previously defined $_host_l{$field[0]}=int($field[4]); if ($withupdate) { # field[5] field[6] are used only for update if ($field[5] && ! $_host_s{$field[0]}) { $_host_s{$field[0]}=int($field[5]); } if ($field[6] && ! $_host_u{$field[0]}) { $_host_u{$field[0]}=$field[6]; } } } $countloaded++; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_VISITOR' || $field[0] eq "${xmleb}END_VISITOR" || ! $_); if ($field[0] ne 'END_VISITOR' && $field[0] ne "${xmleb}END_VISITOR") { error("History file \"$filetoread\" is corrupted (End of section VISITOR not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of VISITOR section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'visitor'}; # WE DO NOT SAVE SECTION NOW TO BE SURE TO HAVE THIS LARGE SECTION NOT AT THE BEGINNING OF FILE #if ($SectionsToSave{'visitor'}) { # Save_History('visitor',$year,$month); delete $SectionsToSave{'visitor'}; # if ($withpurge) { %_host_p=(); %_host_h=(); %_host_k=(); %_host_l=(); %_host_s=(); %_host_u=(); } #} if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_UNKNOWNIP for backward compatibility if ($field[0] eq 'BEGIN_UNKNOWNIP') { my %iptomigrate=(); if ($Debug) { debug(" Begin of UNKNOWNIP section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'unknownip'}) { $iptomigrate{$field[0]}=$field[1]||0; $countloaded++; } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_UNKNOWNIP' || $field[0] eq "${xmleb}END_UNKNOWNIP" || ! $_); if ($field[0] ne 'END_UNKNOWNIP' && $field[0] ne "${xmleb}END_UNKNOWNIP") { error("History file \"$filetoread\" is corrupted (End of section UNKOWNIP not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of UNKOWNIP section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'visitor'}; # THIS SECTION IS NEVER SAVED. ONLY READ FOR MIGRATE AND CONVERTED INTO VISITOR SECTION foreach (keys %iptomigrate) { $_host_p{$_}+=int($_host_p{'Unknown'}/$countloaded); $_host_h{$_}+=int($_host_h{'Unknown'}/$countloaded); $_host_k{$_}+=int($_host_k{'Unknown'}/$countloaded); if ($iptomigrate{$_} > 0) { $_host_l{$_}=$iptomigrate{$_} }; } delete $_host_p{'Unknown'}; delete $_host_h{'Unknown'}; delete $_host_k{'Unknown'}; delete $_host_l{'Unknown'}; if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_LOGIN if ($field[0] eq 'BEGIN_LOGIN') { if ($Debug) { debug(" Begin of LOGIN section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'login'}) { $countloaded++; if ($field[1]) { $_login_p{$field[0]}+=$field[1]; } if ($field[2]) { $_login_h{$field[0]}+=$field[2]; } if ($field[3]) { $_login_k{$field[0]}+=$field[3]; } if (! $_login_l{$field[0]} && $field[4]) { $_login_l{$field[0]}=int($field[4]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_LOGIN' || $field[0] eq "${xmleb}END_LOGIN" || ! $_); if ($field[0] ne 'END_LOGIN' && $field[0] ne "${xmleb}END_LOGIN") { error("History file \"$filetoread\" is corrupted (End of section LOGIN not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of LOGIN section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'login'}; if ($SectionsToSave{'login'}) { Save_History('login',$year,$month); delete $SectionsToSave{'login'}; if ($withpurge) { %_login_p=(); %_login_h=(); %_login_k=(); %_login_l=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_DOMAIN if ($field[0] eq 'BEGIN_DOMAIN') { if ($Debug) { debug(" Begin of DOMAIN section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'domain'}) { $countloaded++; if ($field[1]) { $_domener_p{$field[0]}+=$field[1]; } if ($field[2]) { $_domener_h{$field[0]}+=$field[2]; } if ($field[3]) { $_domener_k{$field[0]}+=$field[3]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_DOMAIN' || $field[0] eq "${xmleb}END_DOMAIN" || ! $_); if ($field[0] ne 'END_DOMAIN' && $field[0] ne "${xmleb}END_DOMAIN") { error("History file \"$filetoread\" is corrupted (End of section DOMAIN not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of DOMAIN section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'domain'}; if ($SectionsToSave{'domain'}) { Save_History('domain',$year,$month); delete $SectionsToSave{'domain'}; if ($withpurge) { %_domener_p=(); %_domener_h=(); %_domener_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_SESSION if ($field[0] eq 'BEGIN_SESSION') { if ($Debug) { debug(" Begin of SESSION section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'session'}) { $countloaded++; if ($field[1]) { $_session{$field[0]}+=$field[1]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_SESSION' || $field[0] eq "${xmleb}END_SESSION" || ! $_); if ($field[0] ne 'END_SESSION' && $field[0] ne "${xmleb}END_SESSION") { error("History file \"$filetoread\" is corrupted (End of section SESSION not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of SESSION section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'session'}; # WE DO NOT SAVE SECTION NOW BECAUSE VALUES CAN BE CHANGED AFTER READING VISITOR #if ($SectionsToSave{'session'}) { # Save_History('session',$year,$month); delete $SectionsToSave{'session'}; } # if ($withpurge) { %_session=(); } #} if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_OS if ($field[0] eq 'BEGIN_OS') { if ($Debug) { debug(" Begin of OS section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'os'}) { $countloaded++; if ($field[1]) { $_os_h{$field[0]}+=$field[1]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_OS' || $field[0] eq "${xmleb}END_OS" || ! $_); if ($field[0] ne 'END_OS' && $field[0] ne "${xmleb}END_OS") { error("History file \"$filetoread\" is corrupted (End of section OS not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of OS section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'os'}; if ($SectionsToSave{'os'}) { Save_History('os',$year,$month); delete $SectionsToSave{'os'}; if ($withpurge) { %_os_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_BROWSER if ($field[0] eq 'BEGIN_BROWSER') { if ($Debug) { debug(" Begin of BROWSER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'browser'}) { $countloaded++; if ($field[1]) { $_browser_h{$field[0]}+=$field[1]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_BROWSER' || $field[0] eq "${xmleb}END_BROWSER" || ! $_); if ($field[0] ne 'END_BROWSER' && $field[0] ne "${xmleb}END_BROWSER") { error("History file \"$filetoread\" is corrupted (End of section BROWSER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of BROWSER section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'browser'}; if ($SectionsToSave{'browser'}) { Save_History('browser',$year,$month); delete $SectionsToSave{'browser'}; if ($withpurge) { %_browser_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_UNKNOWNREFERER if ($field[0] eq 'BEGIN_UNKNOWNREFERER') { if ($Debug) { debug(" Begin of UNKNOWNREFERER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'unknownreferer'}) { $countloaded++; if (! $_unknownreferer_l{$field[0]}) { $_unknownreferer_l{$field[0]}=int($field[1]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_UNKNOWNREFERER' || $field[0] eq "${xmleb}END_UNKNOWNREFERER" || ! $_); if ($field[0] ne 'END_UNKNOWNREFERER' && $field[0] ne "${xmleb}END_UNKNOWNREFERER") { error("History file \"$filetoread\" is corrupted (End of section UNKNOWNREFERER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of UNKNOWNREFERER section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'unknownreferer'}; if ($SectionsToSave{'unknownreferer'}) { Save_History('unknownreferer',$year,$month); delete $SectionsToSave{'unknownreferer'}; if ($withpurge) { %_unknownreferer_l=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_UNKNOWNREFERERBROWSER if ($field[0] eq 'BEGIN_UNKNOWNREFERERBROWSER') { if ($Debug) { debug(" Begin of UNKNOWNREFERERBROWSER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'unknownrefererbrowser'}) { $countloaded++; if (! $_unknownrefererbrowser_l{$field[0]}) { $_unknownrefererbrowser_l{$field[0]}=int($field[1]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_UNKNOWNREFERERBROWSER' || $field[0] eq "${xmleb}END_UNKNOWNREFERERBROWSER" || ! $_); if ($field[0] ne 'END_UNKNOWNREFERERBROWSER' && $field[0] ne "${xmleb}END_UNKNOWNREFERERBROWSER") { error("History file \"$filetoread\" is corrupted (End of section UNKNOWNREFERERBROWSER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of UNKNOWNREFERERBROWSER section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'unknownrefererbrowser'}; if ($SectionsToSave{'unknownrefererbrowser'}) { Save_History('unknownrefererbrowser',$year,$month); delete $SectionsToSave{'unknownrefererbrowser'}; if ($withpurge) { %_unknownrefererbrowser_l=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_SCREENSIZE if ($field[0] eq 'BEGIN_SCREENSIZE') { if ($Debug) { debug(" Begin of SCREENSIZE section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'screensize'}) { $countloaded++; if ($field[1]) { $_screensize_h{$field[0]}+=$field[1]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_SCREENSIZE' || $field[0] eq "${xmleb}END_SCREENSIZE" || ! $_); if ($field[0] ne 'END_SCREENSIZE' && $field[0] ne "${xmleb}END_SCREENSIZE") { error("History file \"$filetoread\" is corrupted (End of section SCREENSIZE not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of SCREENSIZE section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'screensize'}; if ($SectionsToSave{'screensize'}) { Save_History('screensize',$year,$month); delete $SectionsToSave{'screensize'}; if ($withpurge) { %_screensize_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_ROBOT if ($field[0] eq 'BEGIN_ROBOT') { if ($Debug) { debug(" Begin of ROBOT section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'robot'}) { $countloaded++; if ($field[1]) { $_robot_h{$field[0]}+=$field[1]; } if ($versionnum < 5000 || ! $field[3]) { # For backward compatibility if (! $_robot_l{$field[0]}) { $_robot_l{$field[0]}=int($field[2]); } } else { $_robot_k{$field[0]}+=$field[2]; if (! $_robot_l{$field[0]}) { $_robot_l{$field[0]}=int($field[3]); } } if ($field[4]) { $_robot_r{$field[0]}+=$field[4]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_ROBOT' || $field[0] eq "${xmleb}END_ROBOT" || ! $_); if ($field[0] ne 'END_ROBOT' && $field[0] ne "${xmleb}END_ROBOT") { error("History file \"$filetoread\" is corrupted (End of section ROBOT not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of ROBOT section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'robot'}; if ($SectionsToSave{'robot'}) { Save_History('robot',$year,$month); delete $SectionsToSave{'robot'}; if ($withpurge) { %_robot_h=(); %_robot_k=(); %_robot_l=(); %_robot_r=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_WORMS if ($field[0] eq 'BEGIN_WORMS') { if ($Debug) { debug(" Begin of WORMS section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'worms'}) { $countloaded++; if ($field[1]) { $_worm_h{$field[0]}+=$field[1]; } $_worm_k{$field[0]}+=$field[2]; if (! $_worm_l{$field[0]}) { $_worm_l{$field[0]}=int($field[3]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_WORMS' || $field[0] eq "${xmleb}END_WORMS" || ! $_); if ($field[0] ne 'END_WORMS' && $field[0] ne "${xmleb}END_WORMS") { error("History file \"$filetoread\" is corrupted (End of section WORMS not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of WORMS section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'worms'}; if ($SectionsToSave{'worms'}) { Save_History('worms',$year,$month); delete $SectionsToSave{'worms'}; if ($withpurge) { %_worm_h=(); %_worm_k=(); %_worm_l=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_EMAILS if ($field[0] eq 'BEGIN_EMAILSENDER') { if ($Debug) { debug(" Begin of EMAILSENDER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'emailsender'}) { $countloaded++; if ($field[1]) { $_emails_h{$field[0]}+=$field[1]; } if ($field[2]) { $_emails_k{$field[0]}+=$field[2]; } if (! $_emails_l{$field[0]}) { $_emails_l{$field[0]}=int($field[3]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_EMAILSENDER' || $field[0] eq "${xmleb}END_EMAILSENDER" || ! $_); if ($field[0] ne 'END_EMAILSENDER' && $field[0] ne "${xmleb}END_EMAILSENDER") { error("History file \"$filetoread\" is corrupted (End of section EMAILSENDER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of EMAILSENDER section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'emailsender'}; if ($SectionsToSave{'emailsender'}) { Save_History('emailsender',$year,$month); delete $SectionsToSave{'emailsender'}; if ($withpurge) { %_emails_h=(); %_emails_k=(); %_emails_l=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_EMAILR if ($field[0] eq 'BEGIN_EMAILRECEIVER') { if ($Debug) { debug(" Begin of EMAILRECEIVER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'emailreceiver'}) { $countloaded++; if ($field[1]) { $_emailr_h{$field[0]}+=$field[1]; } if ($field[2]) { $_emailr_k{$field[0]}+=$field[2]; } if (! $_emailr_l{$field[0]}) { $_emailr_l{$field[0]}=int($field[3]); } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_EMAILRECEIVER' || $field[0] eq "${xmleb}END_EMAILRECEIVER" || ! $_); if ($field[0] ne 'END_EMAILRECEIVER' && $field[0] ne "${xmleb}END_EMAILRECEIVER") { error("History file \"$filetoread\" is corrupted (End of section EMAILRECEIVER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of EMAILRECEIVER section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'emailreceiver'}; if ($SectionsToSave{'emailreceiver'}) { Save_History('emailreceiver',$year,$month); delete $SectionsToSave{'emailreceiver'}; if ($withpurge) { %_emailr_h=(); %_emailr_k=(); %_emailr_l=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_SIDER if ($field[0] eq 'BEGIN_SIDER') { if ($Debug) { debug(" Begin of SIDER section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'sider'}) { my $loadrecord=0; if ($withupdate) { $loadrecord=1; } else { if ($HTMLOutput{'main'}) { if ($MonthRequired eq 'all') { $loadrecord=1; } else { if ($countloaded < $MaxNbOf{'PageShown'} && $field[1] >= $MinHit{'File'}) { $loadrecord=1; } $TotalDifferentPages++; } } else { # This is for $HTMLOutput = urldetail, urlentry or urlexit if ($MonthRequired eq 'all' ) { if ((!$FilterIn{'url'} || $field[0] =~ /$FilterIn{'url'}/) && (!$FilterEx{'url'} || $field[0] !~ /$FilterEx{'url'}/)) { $loadrecord=1; } } else { if ((!$FilterIn{'url'} || $field[0] =~ /$FilterIn{'url'}/) && (!$FilterEx{'url'} || $field[0] !~ /$FilterEx{'url'}/) && $field[1] >= $MinHit{'File'}) { $loadrecord=1; } $TotalDifferentPages++; } } # Posssibilite de mettre if ($FilterIn{'url'} && $field[0] =~ /$FilterIn{'url'}/) mais il faut gerer TotalPages de la meme maniere if ($versionnum < 4000) { # For history files < 4.0 $TotalEntries+=($field[2]||0); } else { $TotalBytesPages+=($field[2]||0); $TotalEntries+=($field[3]||0); $TotalExits+=($field[4]||0); } } if ($loadrecord) { if ($field[1]) { $_url_p{$field[0]}+=$field[1]; } if ($versionnum < 4000) { # For history files < 4.0 if ($field[2]) { $_url_e{$field[0]}+=$field[2]; } $_url_k{$field[0]}=0; } else { if ($field[2]) { $_url_k{$field[0]}+=$field[2]; } if ($field[3]) { $_url_e{$field[0]}+=$field[3]; } if ($field[4]) { $_url_x{$field[0]}+=$field[4]; } } $countloaded++; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_SIDER' || $field[0] eq "${xmleb}END_SIDER" || ! $_); if ($field[0] ne 'END_SIDER' && $field[0] ne "${xmleb}END_SIDER") { error("History file \"$filetoread\" is corrupted (End of section SIDER not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of SIDER section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'sider'}; # WE DO NOT SAVE SECTION NOW BECAUSE VALUES CAN BE CHANGED AFTER READING VISITOR #if ($SectionsToSave{'sider'}) { # Save_History('sider',$year,$month); delete $SectionsToSave{'sider'}; # if ($withpurge) { %_url_p=(); %_url_k=(); %_url_e=(); %_url_x=(); } #} if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_FILETYPES if ($field[0] eq 'BEGIN_FILETYPES') { if ($Debug) { debug(" Begin of FILETYPES section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'filetypes'}) { $countloaded++; if ($field[1]) { $_filetypes_h{$field[0]}+=$field[1]; } if ($field[2]) { $_filetypes_k{$field[0]}+=$field[2]; } if ($field[3]) { $_filetypes_gz_in{$field[0]}+=$field[3]; } if ($field[4]) { $_filetypes_gz_out{$field[0]}+=$field[4]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_FILETYPES' || $field[0] eq "${xmleb}END_FILETYPES" || ! $_); if ($field[0] ne 'END_FILETYPES' && $field[0] ne "${xmleb}END_FILETYPES") { error("History file \"$filetoread\" is corrupted (End of section FILETYPES not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of FILETYPES section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'filetypes'}; if ($SectionsToSave{'filetypes'}) { Save_History('filetypes',$year,$month); delete $SectionsToSave{'filetypes'}; if ($withpurge) { %_filetypes_h=(); %_filetypes_k=(); %_filetypes_gz_in=(); %_filetypes_gz_out=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_SEREFERRALS if ($field[0] eq 'BEGIN_SEREFERRALS') { if ($Debug) { debug(" Begin of SEREFERRALS section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'sereferrals'}) { $countloaded++; if ($versionnum < 5004) { # For history files < 5.4 my $se=$field[0]; $se=~s/\./\\./g; if ($SearchEnginesHashID{$se}) { $_se_referrals_h{$SearchEnginesHashID{$se}}+=$field[1]||0; } else { $_se_referrals_h{$field[0]}+=$field[1]||0; } } elsif ($versionnum < 5091) { # For history files < 5.91 my $se=$field[0]; $se=~s/\./\\./g; if ($SearchEnginesHashID{$se}) { $_se_referrals_p{$SearchEnginesHashID{$se}}+=$field[1]||0; $_se_referrals_h{$SearchEnginesHashID{$se}}+=$field[2]||0; } else { $_se_referrals_p{$field[0]}+=$field[1]||0; $_se_referrals_h{$field[0]}+=$field[2]||0; } } else { if ($field[1]) { $_se_referrals_p{$field[0]}+=$field[1]; } if ($field[2]) { $_se_referrals_h{$field[0]}+=$field[2]; } } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_SEREFERRALS' || $field[0] eq "${xmleb}END_SEREFERRALS" || ! $_); if ($field[0] ne 'END_SEREFERRALS' && $field[0] ne "${xmleb}END_SEREFERRALS") { error("History file \"$filetoread\" is corrupted (End of section SEREFERRALS not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of SEREFERRALS section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'sereferrals'}; if ($SectionsToSave{'sereferrals'}) { Save_History('sereferrals',$year,$month); delete $SectionsToSave{'sereferrals'}; if ($withpurge) { %_se_referrals_p=(); %_se_referrals_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_PAGEREFS if ($field[0] eq 'BEGIN_PAGEREFS') { if ($Debug) { debug(" Begin of PAGEREFS section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'pagerefs'}) { my $loadrecord=0; if ($withupdate) { $loadrecord=1; } else { if ((!$FilterIn{'refererpages'} || $field[0] =~ /$FilterIn{'refererpages'}/) && (!$FilterEx{'refererpages'} || $field[0] !~ /$FilterEx{'refererpages'}/)) { $loadrecord=1; } } if ($loadrecord) { if ($versionnum < 5004) { # For history files < 5.4 if ($field[1]) { $_pagesrefs_h{$field[0]}+=int($field[1]); } } else { if ($field[1]) { $_pagesrefs_p{$field[0]}+=int($field[1]); } if ($field[2]) { $_pagesrefs_h{$field[0]}+=int($field[2]); } } $countloaded++; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_PAGEREFS' || $field[0] eq "${xmleb}END_PAGEREFS" || ! $_); if ($field[0] ne 'END_PAGEREFS' && $field[0] ne "${xmleb}END_PAGEREFS") { error("History file \"$filetoread\" is corrupted (End of section PAGEREFS not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of PAGEREFS section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'pagerefs'}; if ($SectionsToSave{'pagerefs'}) { Save_History('pagerefs',$year,$month); delete $SectionsToSave{'pagerefs'}; if ($withpurge) { %_pagesrefs_p=(); %_pagesrefs_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_SEARCHWORDS if ($field[0] eq 'BEGIN_SEARCHWORDS') { if ($Debug) { debug(" Begin of SEARCHWORDS section ($MaxNbOf{'KeyphrasesShown'},$MinHit{'Keyphrase'})"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'searchwords'}) { my $loadrecord=0; if ($withupdate) { $loadrecord=1; } else { if ($HTMLOutput{'main'}) { if ($MonthRequired eq 'all') { $loadrecord=1; } else { if ($countloaded < $MaxNbOf{'KeyphrasesShown'} && $field[1] >= $MinHit{'Keyphrase'}) { $loadrecord=1; } $TotalDifferentKeyphrases++; $TotalKeyphrases+=($field[1]||0); } } elsif ($HTMLOutput{'keyphrases'}) { # Load keyphrases for keyphrases chart if ($MonthRequired eq 'all' ) { $loadrecord=1; } else { if ($field[1] >= $MinHit{'Keyphrase'}) { $loadrecord=1; } $TotalDifferentKeyphrases++; $TotalKeyphrases+=($field[1]||0); } } if ($HTMLOutput{'keywords'}) { # Load keyphrases for keywords chart $loadrecord=2; } } if ($loadrecord) { if ($field[1]) { if ($loadrecord==2) { foreach (split(/\+/,$field[0])) { # field[0] is "val1+val2+..." $_keywords{$_}+=$field[1]; } } else { $_keyphrases{$field[0]}+=$field[1]; } } $countloaded++; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_SEARCHWORDS' || $field[0] eq "${xmleb}END_SEARCHWORDS" || ! $_); if ($field[0] ne 'END_SEARCHWORDS' && $field[0] ne "${xmleb}END_SEARCHWORDS") { error("History file \"$filetoread\" is corrupted (End of section SEARCHWORDS not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of SEARCHWORDS section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'searchwords'}; if ($SectionsToSave{'searchwords'}) { Save_History('searchwords',$year,$month); delete $SectionsToSave{'searchwords'}; # This save searwords and keywords sections if ($withpurge) { %_keyphrases=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_KEYWORDS if ($field[0] eq 'BEGIN_KEYWORDS') { if ($Debug) { debug(" Begin of KEYWORDS section ($MaxNbOf{'KeywordsShown'},$MinHit{'Keyword'})"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'keywords'}) { my $loadrecord=0; if ($MonthRequired eq 'all') { $loadrecord=1; } else { if ($countloaded < $MaxNbOf{'KeywordsShown'} && $field[1] >= $MinHit{'Keyword'}) { $loadrecord=1; } $TotalDifferentKeywords++; $TotalKeywords+=($field[1]||0); } if ($loadrecord) { if ($field[1]) { $_keywords{$field[0]}+=$field[1]; } $countloaded++; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_KEYWORDS' || $field[0] eq "${xmleb}END_KEYWORDS" || ! $_); if ($field[0] ne 'END_KEYWORDS' && $field[0] ne "${xmleb}END_KEYWORDS") { error("History file \"$filetoread\" is corrupted (End of section KEYWORDS not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of KEYWORDS section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'keywords'}; if ($SectionsToSave{'keywords'}) { Save_History('keywords',$year,$month); delete $SectionsToSave{'keywords'}; if ($withpurge) { %_keywords=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_ERRORS if ($field[0] eq 'BEGIN_ERRORS') { if ($Debug) { debug(" Begin of ERRORS section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{'errors'}) { $countloaded++; if ($field[1]) { $_errors_h{$field[0]}+=$field[1]; } if ($field[2]) { $_errors_k{$field[0]}+=$field[2]; } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq 'END_ERRORS' || $field[0] eq "${xmleb}END_ERRORS" || ! $_); if ($field[0] ne 'END_ERRORS' && $field[0] ne "${xmleb}END_ERRORS") { error("History file \"$filetoread\" is corrupted (End of section ERRORS not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of ERRORS section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{'errors'}; if ($SectionsToSave{'errors'}) { Save_History('errors',$year,$month); delete $SectionsToSave{'errors'}; if ($withpurge) { %_errors_h=(); %_errors_k=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } # BEGIN_SIDER_xxx foreach my $code (keys %TrapInfosForHTTPErrorCodes) { if ($field[0] eq "BEGIN_SIDER_$code") { if ($Debug) { debug(" Begin of SIDER_$code section"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0]) { $count++; if ($SectionsToLoad{"sider_$code"}) { $countloaded++; if ($field[1]) { $_sider404_h{$field[0]}+=$field[1]; } if ($withupdate || $HTMLOutput{"errors$code"}) { if ($field[2]) { $_referer404_h{$field[0]}=$field[2]; } } } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq "END_SIDER_$code" || $field[0] eq "${xmleb}END_SIDER_$code" || ! $_); if ($field[0] ne "END_SIDER_$code" && $field[0] ne "${xmleb}END_SIDER_$code") { error("History file \"$filetoread\" is corrupted (End of section SIDER_$code not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of SIDER_$code section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{"sider_$code"}; if ($SectionsToSave{"sider_$code"}) { Save_History("sider_$code",$year,$month); delete $SectionsToSave{"sider_$code"}; if ($withpurge) { %_sider404_h=(); %_referer404_h=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } } # BEGIN_EXTRA_xxx foreach my $extranum (1..@ExtraName-1) { if ($field[0] eq "BEGIN_EXTRA_$extranum") { if ($Debug) { debug(" Begin of EXTRA_$extranum"); } $field[0]=''; my $count=0;my $countloaded=0; do { if ($field[0] ne '') { $count++; if ($SectionsToLoad{"extra_$extranum"}) { if ($ExtraStatTypes[$extranum] =~ /P/i && $field[1]) { ${'_section_' . $extranum . '_p'}{$field[0]}+=$field[1]; } ${'_section_' . $extranum . '_h'}{$field[0]}+=$field[2]; if ($ExtraStatTypes[$extranum] =~ /B/i && $field[3]) { ${'_section_' . $extranum . '_k'}{$field[0]}+=$field[3]; } if ($ExtraStatTypes[$extranum] =~ /L/i && ! ${'_section_' . $extranum . '_l'}{$field[0]} && $field[4]) { ${'_section_' . $extranum . '_l'}{$field[0]}=int($field[4]); } $countloaded++; } } $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq "END_EXTRA_$extranum" || $field[0] eq "${xmleb}END_EXTRA_$extranum" || ! $_); if ($field[0] ne "END_EXTRA_$extranum" && $field[0] ne "${xmleb}END_EXTRA_$extranum") { error("History file \"$filetoread\" is corrupted (End of section EXTRA_$extranum not found).\nRestore a recent backup of this file (data for this month will be restored to backup date), remove it (data for month will be lost), or remove the corrupted section in file (data for at least this section will be lost).","","",1); } if ($Debug) { debug(" End of EXTRA_$extranum section ($count entries, $countloaded loaded)"); } delete $SectionsToLoad{"extra_$extranum"}; if ($SectionsToSave{"extra_$extranum"}) { Save_History("extra_$extranum",$year,$month); delete $SectionsToSave{"extra_$extranum"}; if ($withpurge) { %{'_section_' . $extranum . '_p'}=(); %{'_section_' . $extranum . '_h'}=(); %{'_section_' . $extranum . '_b'}=(); %{'_section_' . $extranum . '_l'}=(); } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } next; } } # BEGIN_PLUGINS if ($AtLeastOneSectionPlugin && $field[0] =~ /^BEGIN_PLUGIN_(\w+)$/i) { my $pluginname=$1; my $found=0; foreach (keys %{$PluginsLoaded{'SectionInitHashArray'}}) { if ($pluginname eq $_) { # The plugin for this section was loaded $found=1; my $issectiontoload=$SectionsToLoad{"plugin_$pluginname"}; my $function="SectionReadHistory_$pluginname(\$issectiontoload,\$xmlold,\$xmleb,\$countlines)"; eval("$function"); delete $SectionsToLoad{"plugin_$pluginname"}; if ($SectionsToSave{"plugin_$pluginname"}) { Save_History("plugin_$pluginname",$year,$month); delete $SectionsToSave{"plugin_$pluginname"}; if ($withpurge) { my $function="SectionInitHashArray_$pluginname()"; eval("$function"); } } last; } } if (! scalar %SectionsToLoad) { debug(" Stop reading history file. Got all we need."); last; } # The plugin for this section was not loaded if (! $found) { do { $_=; chomp $_; s/\r//; @field=split(/\s+/,($xmlold?CleanFromTags($_):$_)); $countlines++; } until ($field[0] eq "END_PLUGIN_$pluginname" || $field[0] eq "${xmleb}END_PLUGIN_$pluginname" || ! $_); } next; } # For backward compatibility (ORIGIN section was "HitFromx" in old history files) if ($SectionsToLoad{'origin'}) { if ($field[0] eq 'HitFrom0') { $_from_p[0]+=0; $_from_h[0]+=$field[1]; next; } if ($field[0] eq 'HitFrom1') { $_from_p[1]+=0; $_from_h[1]+=$field[1]; next; } if ($field[0] eq 'HitFrom2') { $_from_p[2]+=0; $_from_h[2]+=$field[1]; next; } if ($field[0] eq 'HitFrom3') { $_from_p[3]+=0; $_from_h[3]+=$field[1]; next; } if ($field[0] eq 'HitFrom4') { $_from_p[4]+=0; $_from_h[4]+=$field[1]; next; } if ($field[0] eq 'HitFrom5') { $_from_p[5]+=0; $_from_h[5]+=$field[1]; next; } } } } if ($withupdate) { # Process rest of data saved in 'wait' arrays (data for hosts that are not in history file or no history file found) # This can change some values for day, sider and session sections if ($Debug) { debug(" Processing data in 'wait' arrays",3); } foreach (keys %_waithost_e) { if ($Debug) { debug(" Visit in 'wait' array for $_ is a new visit",4); } my $newtimehosts=($_waithost_s{$_}?$_waithost_s{$_}:$_host_s{$_}); my $newtimehostl=($_waithost_l{$_}?$_waithost_l{$_}:$_host_l{$_}); $_url_e{$_waithost_e{$_}}++; $newtimehosts =~ /^(\d\d\d\d\d\d\d\d)/; $DayVisits{$1}++; if ($_waithost_s{$_}) { # There was also a second session in processed log $_session{GetSessionRange($newtimehosts,$newtimehostl)}++; } } } # Write all unwrote sections in section order ('general','time', 'day','sider','session' and other...) foreach my $key (sort { $SectionsToSave{$a} <=> $SectionsToSave{$b} } keys %SectionsToSave) { Save_History("$key",$year,$month,$lastlinenb,$lastlineoffset,$lastlinechecksum); } # E-lane-->Save new petitions section Save_History("petitions",$year,$month); %SectionsToSave=(); # Update offset in map section and last data in general section then close files if ($withupdate) { if ($xml) { print HISTORYTMP "\n\n\n"; } # Update offset of sections in the MAP section foreach (sort { $PosInFile{$a} <=> $PosInFile{$b} } keys %ValueInFile) { if ($Debug) { debug(" Update offset of section $_=$ValueInFile{$_} in file at offset $PosInFile{$_}"); } if ($PosInFile{"$_"}) { seek(HISTORYTMP,$PosInFile{"$_"},0); print HISTORYTMP $ValueInFile{"$_"}; } } # Save last data in general sections if ($Debug) { debug(" Update MonthVisits=$MonthVisits{$year.$month} in file at offset $PosInFile{TotalVisits}"); } seek(HISTORYTMP,$PosInFile{"TotalVisits"},0); print HISTORYTMP $MonthVisits{$year.$month}; if ($Debug) { debug(" Update MonthUnique=$MonthUnique{$year.$month} in file at offset $PosInFile{TotalUnique}"); } seek(HISTORYTMP,$PosInFile{"TotalUnique"},0); print HISTORYTMP $MonthUnique{$year.$month}; if ($Debug) { debug(" Update MonthHostsKnown=$MonthHostsKnown{$year.$month} in file at offset $PosInFile{MonthHostsKnown}"); } seek(HISTORYTMP,$PosInFile{"MonthHostsKnown"},0); print HISTORYTMP $MonthHostsKnown{$year.$month}; if ($Debug) { debug(" Update MonthHostsUnknown=$MonthHostsUnknown{$year.$month} in file at offset $PosInFile{MonthHostsUnknown}"); } seek(HISTORYTMP,$PosInFile{"MonthHostsUnknown"},0); print HISTORYTMP $MonthHostsUnknown{$year.$month}; close(HISTORYTMP) || error("Failed to write temporary history file"); } if ($withread) { close(HISTORY) || error("Command for pipe '$filetoread' failed"); } # Purge data if ($withpurge) { &Init_HashArray(); } # If update, rename tmp file bis into tmp file or set HistoryAlreadyFlushed if ($withupdate) { if ($HistoryAlreadyFlushed{"$year$month"}) { debug("Rename tmp history file bis '$filetoread' to '$filetowrite'"); if (rename($filetowrite,$filetoread)==0) { error("Failed to update tmp history file $filetoread"); } } else { $HistoryAlreadyFlushed{"$year$month"}=1; } if (! $ListOfYears{"$year"} || $ListOfYears{"$year"} lt "$month") { $ListOfYears{"$year"}="$month"; } } # For backward compatibility, if LastLine does not exist, set to LastTime $LastLine||=$LastTime{$year.$month}; return ($withupdate?"$filetowrite":""); } #------------------------------------------------------------------------------ # Function: Save a part of history file # Parameters: sectiontosave,year,month[,lastlinenb,lastlineoffset,lastlinechecksum] # Input: $VERSION HISTORYTMP $nowyear $nowmonth $nowday $nowhour $nowmin $nowsec $LastLineNumber $LastLineOffset $LastLineChecksum # Output: None # Return: None #------------------------------------------------------------------------------ sub Save_History { my $sectiontosave=shift||''; my $year=shift||''; my $month=shift||''; my $xml=($BuildHistoryFormat eq 'xml'?1:0); my ($xmlbb,$xmlbs,$xmlbe,$xmlhb,$xmlhs,$xmlhe,$xmlrb,$xmlrs,$xmlre,$xmleb,$xmlee)=('','','','','','','','','','',''); if ($xml) { ($xmlbb,$xmlbs,$xmlbe,$xmlhb,$xmlhs,$xmlhe,$xmlrb,$xmlrs,$xmlre,$xmleb,$xmlee)=("\n",'','','','','
','','
','','
',"\n" ); } else { $xmlbs=' '; $xmlhs=' '; $xmlrs=' '; } my $lastlinenb=shift||0; my $lastlineoffset=shift||0; my $lastlinechecksum=shift||0; if (! $lastlinenb) { # This happens for migrate $lastlinenb=$LastLineNumber; $lastlineoffset=$LastLineOffset; $lastlinechecksum=$LastLineChecksum; } if ($Debug) { debug(" Save_History [sectiontosave=$sectiontosave,year=$year,month=$month,lastlinenb=$lastlinenb,lastlineoffset=$lastlineoffset,lastlinechecksum=$lastlinechecksum]",1); } my $spacebar=" "; my %keysinkeylist=(); # Header if ($sectiontosave eq 'header') { if ($xml) { print HISTORYTMP "\n"; } print HISTORYTMP "AWSTATS DATA FILE $VERSION\n"; if ($xml) { print HISTORYTMP "\n"; } print HISTORYTMP "# If you remove this file, all statistics for date $year-$month will be lost/reset.\n"; if ($xml) { print HISTORYTMP "\n"; } print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Position (offset in bytes) in this file of beginning of each section for\n"; print HISTORYTMP "# direct I/O access. If you made changes somewhere in this file, you should\n"; print HISTORYTMP "# also remove completely the MAP section (AWStats will rewrite it at next\n"; print HISTORYTMP "# update).\n"; print HISTORYTMP "${xmlbb}BEGIN_MAP${xmlbs}".(26+(scalar keys %TrapInfosForHTTPErrorCodes)+(scalar @ExtraName?scalar @ExtraName-1:0)+(scalar keys %{$PluginsLoaded{'SectionInitHashArray'}}))."${xmlbe}\n"; print HISTORYTMP "${xmlrb}POS_GENERAL${xmlrs}";$PosInFile{"general"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; # When print HISTORYTMP "${xmlrb}POS_TIME${xmlrs}";$PosInFile{"time"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_VISITOR${xmlrs}";$PosInFile{"visitor"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_DAY${xmlrs}";$PosInFile{"day"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; # Who print HISTORYTMP "${xmlrb}POS_DOMAIN${xmlrs}";$PosInFile{"domain"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_LOGIN${xmlrs}";$PosInFile{"login"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_ROBOT${xmlrs}";$PosInFile{"robot"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_WORMS${xmlrs}";$PosInFile{"worms"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_EMAILSENDER${xmlrs}";$PosInFile{"emailsender"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_EMAILRECEIVER${xmlrs}";$PosInFile{"emailreceiver"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; # Navigation print HISTORYTMP "${xmlrb}POS_SESSION${xmlrs}";$PosInFile{"session"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_SIDER${xmlrs}";$PosInFile{"sider"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_FILETYPES${xmlrs}";$PosInFile{"filetypes"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_OS${xmlrs}";$PosInFile{"os"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_BROWSER${xmlrs}";$PosInFile{"browser"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_SCREENSIZE${xmlrs}";$PosInFile{"screensize"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_UNKNOWNREFERER${xmlrs}";$PosInFile{'unknownreferer'}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_UNKNOWNREFERERBROWSER${xmlrs}";$PosInFile{'unknownrefererbrowser'}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; # Referers print HISTORYTMP "${xmlrb}POS_ORIGIN${xmlrs}";$PosInFile{"origin"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_SEREFERRALS${xmlrs}";$PosInFile{"sereferrals"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_PAGEREFS${xmlrs}";$PosInFile{"pagerefs"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_SEARCHWORDS${xmlrs}";$PosInFile{"searchwords"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_KEYWORDS${xmlrs}";$PosInFile{"keywords"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; # Others print HISTORYTMP "${xmlrb}POS_MISC${xmlrs}";$PosInFile{"misc"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_ERRORS${xmlrs}";$PosInFile{"errors"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}POS_CLUSTER${xmlrs}";$PosInFile{"cluster"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; foreach (keys %TrapInfosForHTTPErrorCodes) { print HISTORYTMP "${xmlrb}POS_SIDER_$_${xmlrs}";$PosInFile{"sider_$_"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; } foreach (1..@ExtraName-1) { print HISTORYTMP "${xmlrb}POS_EXTRA_$_${xmlrs}";$PosInFile{"extra_$_"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; } foreach (keys %{$PluginsLoaded{'SectionInitHashArray'}}) { print HISTORYTMP "${xmlrb}POS_PLUGIN_$_${xmlrs}";$PosInFile{"plugin_$_"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; } print HISTORYTMP "${xmleb}END_MAP${xmlee}\n"; } # General if ($sectiontosave eq 'general') { if ($LastUpdate < int("$nowyear$nowmonth$nowday$nowhour$nowmin$nowsec")) { $LastUpdate=int("$nowyear$nowmonth$nowday$nowhour$nowmin$nowsec"); } print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# LastLine = Date of last record processed - Last record line number in last log - Last record offset in last log - Last record signature value\n"; print HISTORYTMP "# FirstTime = Date of first visit for history file\n"; print HISTORYTMP "# LastTime = Date of last visit for history file\n"; print HISTORYTMP "# LastUpdate = Date of last update - Nb of parsed records - Nb of parsed old records - Nb of parsed new records - Nb of parsed corrupted - Nb of parsed dropped\n"; print HISTORYTMP "# TotalVisits = Number of visits\n"; print HISTORYTMP "# TotalUnique = Number of unique visitors\n"; print HISTORYTMP "# MonthHostsKnown = Number of hosts known\n"; print HISTORYTMP "# MonthHostsUnKnown = Number of hosts unknown\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_GENERAL${xmlbs}8${xmlbe}\n"; print HISTORYTMP "${xmlrb}LastLine${xmlrs}".($LastLine>0?$LastLine:$LastTime{$year.$month})." $lastlinenb $lastlineoffset $lastlinechecksum${xmlre}\n"; print HISTORYTMP "${xmlrb}FirstTime${xmlrs}$FirstTime{$year.$month}${xmlre}\n"; print HISTORYTMP "${xmlrb}LastTime${xmlrs}$LastTime{$year.$month}${xmlre}\n"; print HISTORYTMP "${xmlrb}LastUpdate${xmlrs}$LastUpdate $NbOfLinesParsed $NbOfOldLines $NbOfNewLines $NbOfLinesCorrupted $NbOfLinesDropped${xmlre}\n"; print HISTORYTMP "${xmlrb}TotalVisits${xmlrs}";$PosInFile{"TotalVisits"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}TotalUnique${xmlrs}";$PosInFile{"TotalUnique"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}MonthHostsKnown${xmlrs}";$PosInFile{"MonthHostsKnown"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmlrb}MonthHostsUnknown${xmlrs}";$PosInFile{"MonthHostsUnknown"}=tell HISTORYTMP;print HISTORYTMP "$spacebar${xmlre}\n"; print HISTORYTMP "${xmleb}".(${xmleb}?"\n":"")."END_GENERAL${xmlee}\n"; # END_GENERAL on a new line following xml tag because END_ detection does not work like other sections } # When if ($sectiontosave eq 'time') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Hour - Pages - Hits - Bandwidth - Not viewed Pages - Not viewed Hits - Not viewed Bandwidth\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_TIME${xmlbs}24${xmlbe}\n"; for (my $ix=0; $ix<=23; $ix++) { print HISTORYTMP "${xmlrb}$ix${xmlrs}".int($_time_p[$ix])."${xmlrs}".int($_time_h[$ix])."${xmlrs}".int($_time_k[$ix])."${xmlrs}".int($_time_nv_p[$ix])."${xmlrs}".int($_time_nv_h[$ix])."${xmlrs}".int($_time_nv_k[$ix])."${xmlre}\n"; } print HISTORYTMP "${xmleb}END_TIME${xmlee}\n"; } if ($sectiontosave eq 'day') { # This section must be saved after VISITOR section is read print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Date - Pages - Hits - Bandwidth - Visits\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_DAY${xmlbs}".(scalar keys %DayHits)."${xmlbe}\n"; my $monthvisits=0; foreach (sort keys %DayHits) { if ($_ =~ /^$year$month/i) { # Found a day entry of the good month my $page=$DayPages{$_}||0; my $hits=$DayHits{$_}||0; my $bytes=$DayBytes{$_}||0; my $visits=$DayVisits{$_}||0; print HISTORYTMP "${xmlrb}$_${xmlrs}$page${xmlrs}$hits${xmlrs}$bytes${xmlrs}$visits${xmlre}\n"; $monthvisits+=$visits; } } $MonthVisits{$year.$month}=$monthvisits; print HISTORYTMP "${xmleb}END_DAY${xmlee}\n"; } # Who if ($sectiontosave eq 'domain') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'Domain'}\n"; } print HISTORYTMP "# Domain - Pages - Hits - Bandwidth\n"; print HISTORYTMP "# The $MaxNbOf{'Domain'} first Pages must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_DOMAIN${xmlbs}".(scalar keys %_domener_h)."${xmlbe}\n"; # We save page list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'Domain'},$MinHit{'Domain'},\%_domener_h,\%_domener_p); my %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; my $page=$_domener_p{$_}||0; my $bytes=$_domener_k{$_}||0; # ||0 could be commented to reduce history file size print HISTORYTMP "${xmlrb}$_${xmlrs}$page${xmlrs}$_domener_h{$_}${xmlrs}$bytes${xmlre}\n"; } foreach (keys %_domener_h) { if ($keysinkeylist{$_}) { next; } my $page=$_domener_p{$_}||0; my $bytes=$_domener_k{$_}||0; # ||0 could be commented to reduce history file size print HISTORYTMP "${xmlrb}$_${xmlrs}$page${xmlrs}$_domener_h{$_}${xmlrs}$bytes${xmlre}\n"; } print HISTORYTMP "${xmleb}END_DOMAIN${xmlee}\n"; } if ($sectiontosave eq 'visitor') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'HostsShown'}\n"; } print HISTORYTMP "# Host - Pages - Hits - Bandwidth - Last visit date - [Start date of last visit] - [Last page of last visit]\n"; print HISTORYTMP "# [Start date of last visit] and [Last page of last visit] are saved only if session is not finished\n"; print HISTORYTMP "# The $MaxNbOf{'HostsShown'} first Hits must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_VISITOR${xmlbs}".(scalar keys %_host_h)."${xmlbe}\n"; my $monthhostsknown=0; # We save page list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'HostsShown'},$MinHit{'Host'},\%_host_h,\%_host_p); my %keysinkeylist=(); foreach my $key (@keylist) { if ($key !~ /^\d+\.\d+\.\d+\.\d+$/ && $key !~ /^[0-9A-F]*:/i) { $monthhostsknown++; } $keysinkeylist{$key}=1; my $page=$_host_p{$key}||0; my $bytes=$_host_k{$key}||0; my $timehostl=$_host_l{$key}||0; my $timehosts=$_host_s{$key}||0; my $lastpage=$_host_u{$key}||''; if ($timehostl && $timehosts && $lastpage) { if (($timehostl+$VISITTIMEOUT) < $LastLine) { # Session for this user is expired if ($timehosts) { $_session{GetSessionRange($timehosts,$timehostl)}++; } if ($lastpage) { $_url_x{$lastpage}++; } delete $_host_s{$key}; delete $_host_u{$key}; print HISTORYTMP "${xmlrb}$key${xmlrs}$page${xmlrs}$_host_h{$key}${xmlrs}$bytes${xmlrs}$timehostl${xmlre}\n"; } else { # If this user has started a new session that is not expired print HISTORYTMP "${xmlrb}$key${xmlrs}$page${xmlrs}$_host_h{$key}${xmlrs}$bytes${xmlrs}$timehostl${xmlrs}$timehosts${xmlrs}$lastpage${xmlre}\n"; } } else { my $hostl=$timehostl||''; print HISTORYTMP "${xmlrb}$key${xmlrs}$page${xmlrs}$_host_h{$key}${xmlrs}$bytes${xmlrs}$hostl${xmlre}\n"; } } foreach my $key (keys %_host_h) { if ($keysinkeylist{$key}) { next; } if ($key !~ /^\d+\.\d+\.\d+\.\d+$/ && $key !~ /^[0-9A-F]*:/i) { $monthhostsknown++; } my $page=$_host_p{$key}||0; my $bytes=$_host_k{$key}||0; my $timehostl=$_host_l{$key}||0; my $timehosts=$_host_s{$key}||0; my $lastpage=$_host_u{$key}||''; if ($timehostl && $timehosts && $lastpage) { if (($timehostl+$VISITTIMEOUT) < $LastLine) { # Session for this user is expired if ($timehosts) { $_session{GetSessionRange($timehosts,$timehostl)}++; } if ($lastpage) { $_url_x{$lastpage}++; } delete $_host_s{$key}; delete $_host_u{$key}; print HISTORYTMP "${xmlrb}$key${xmlrs}$page${xmlrs}$_host_h{$key}${xmlrs}$bytes${xmlrs}$timehostl${xmlre}\n"; } else { # If this user has started a new session that is not expired print HISTORYTMP "${xmlrb}$key${xmlrs}$page${xmlrs}$_host_h{$key}${xmlrs}$bytes${xmlrs}$timehostl${xmlrs}$timehosts${xmlrs}$lastpage${xmlre}\n"; } } else { my $hostl=$timehostl||''; print HISTORYTMP "${xmlrb}$key${xmlrs}$page${xmlrs}$_host_h{$key}${xmlrs}$bytes${xmlrs}$hostl${xmlre}\n"; } } $MonthUnique{$year.$month}=(scalar keys %_host_p); $MonthHostsKnown{$year.$month}=$monthhostsknown; $MonthHostsUnknown{$year.$month}=(scalar keys %_host_h) - $monthhostsknown; print HISTORYTMP "${xmleb}END_VISITOR${xmlee}\n"; } if ($sectiontosave eq 'login') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'LoginShown'}\n"; } print HISTORYTMP "# Login - Pages - Hits - Bandwidth - Last visit\n"; print HISTORYTMP "# The $MaxNbOf{'LoginShown'} first Pages must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_LOGIN${xmlbs}".(scalar keys %_login_h)."${xmlbe}\n"; # We save login list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'LoginShown'},$MinHit{'Login'},\%_login_h,\%_login_p); my %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_login_p{$_}||0)."${xmlrs}".int($_login_h{$_}||0)."${xmlrs}".int($_login_k{$_}||0)."${xmlrs}".($_login_l{$_}||'')."${xmlre}\n"; } foreach (keys %_login_h) { if ($keysinkeylist{$_}) { next; } print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_login_p{$_}||0)."${xmlrs}".int($_login_h{$_}||0)."${xmlrs}".int($_login_k{$_}||0)."${xmlrs}".($_login_l{$_}||'')."${xmlre}\n"; } print HISTORYTMP "${xmleb}END_LOGIN${xmlee}\n"; } if ($sectiontosave eq 'robot') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'RobotShown'}\n"; } print HISTORYTMP "# Robot ID - Hits - Bandwidth - Last visit - Hits on robots.txt\n"; print HISTORYTMP "# The $MaxNbOf{'RobotShown'} first Hits must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_ROBOT${xmlbs}".(scalar keys %_robot_h)."${xmlbe}\n"; # We save robot list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'RobotShown'},$MinHit{'Robot'},\%_robot_h,\%_robot_h); my %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_robot_h{$_})."${xmlrs}".int($_robot_k{$_})."${xmlrs}$_robot_l{$_}${xmlrs}".int($_robot_r{$_})."${xmlre}\n"; } foreach (keys %_robot_h) { if ($keysinkeylist{$_}) { next; } print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_robot_h{$_})."${xmlrs}".int($_robot_k{$_})."${xmlrs}$_robot_l{$_}${xmlrs}".int($_robot_r{$_})."${xmlre}\n"; } print HISTORYTMP "${xmleb}END_ROBOT${xmlee}\n"; } if ($sectiontosave eq 'worms') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'WormsShown'}\n"; } print HISTORYTMP "# Worm ID - Hits - Bandwidth - Last visit\n"; print HISTORYTMP "# The $MaxNbOf{'WormsShown'} first Hits must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_WORMS${xmlbs}".(scalar keys %_worm_h)."${xmlbe}\n"; # We save worm list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'WormsShown'},$MinHit{'Worm'},\%_worm_h,\%_worm_h); my %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_worm_h{$_})."${xmlrs}".int($_worm_k{$_})."${xmlrs}$_worm_l{$_}${xmlre}\n"; } foreach (keys %_worm_h) { if ($keysinkeylist{$_}) { next; } print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_worm_h{$_})."${xmlrs}".int($_worm_k{$_})."${xmlrs}$_worm_l{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_WORMS${xmlee}\n"; } if ($sectiontosave eq 'emailsender') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'EMailsShown'}\n"; } print HISTORYTMP "# EMail - Hits - Bandwidth - Last visit\n"; print HISTORYTMP "# The $MaxNbOf{'EMailsShown'} first Hits must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_EMAILSENDER${xmlbs}".(scalar keys %_emails_h)."${xmlbe}\n"; # We save sender email list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'EMailsShown'},$MinHit{'EMail'},\%_emails_h,\%_emails_h); my %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_emails_h{$_}||0)."${xmlrs}".int($_emails_k{$_}||0)."${xmlrs}$_emails_l{$_}${xmlre}\n"; } foreach (keys %_emails_h) { if ($keysinkeylist{$_}) { next; } print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_emails_h{$_}||0)."${xmlrs}".int($_emails_k{$_}||0)."${xmlrs}$_emails_l{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_EMAILSENDER${xmlee}\n"; } if ($sectiontosave eq 'emailreceiver') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'EMailsShown'}\n"; } print HISTORYTMP "# EMail - Hits - Bandwidth - Last visit\n"; print HISTORYTMP "# The $MaxNbOf{'EMailsShown'} first hits must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_EMAILRECEIVER${xmlbs}".(scalar keys %_emailr_h)."${xmlbe}\n"; # We save receiver email list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'EMailsShown'},$MinHit{'EMail'},\%_emailr_h,\%_emailr_h); my %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_emailr_h{$_}||0)."${xmlrs}".int($_emailr_k{$_}||0)."${xmlrs}$_emailr_l{$_}${xmlre}\n"; } foreach (keys %_emailr_h) { if ($keysinkeylist{$_}) { next; } print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_emailr_h{$_}||0)."${xmlrs}".int($_emailr_k{$_}||0)."${xmlrs}$_emailr_l{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_EMAILRECEIVER${xmlee}\n"; } # Navigation if ($sectiontosave eq 'session') { # This section must be saved after VISITOR section is read print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Session range - Number of visits\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_SESSION${xmlbs}".(scalar keys %_session)."${xmlbe}\n"; foreach (keys %_session) { print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_session{$_})."${xmlre}\n"; } print HISTORYTMP "${xmleb}END_SESSION${xmlee}\n"; } if ($sectiontosave eq 'sider') { # This section must be saved after VISITOR section is read print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'PageShown'}\n"; } print HISTORYTMP "# URL - Pages - Bandwidth - Entry - Exit\n"; print HISTORYTMP "# The $MaxNbOf{'PageShown'} first Pages must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_SIDER${xmlbs}".(scalar keys %_url_p)."${xmlbe}\n"; # We save page list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'PageShown'},$MinHit{'File'},\%_url_p,\%_url_p); %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; my $newkey=$_; $newkey =~ s/([^:])\/\//$1\//g; # Because some targeted url were taped with 2 / (Ex: //rep//file.htm). We must keep http://rep/file.htm print HISTORYTMP "${xmlrb}$newkey${xmlrs}".int($_url_p{$_}||0)."${xmlrs}".int($_url_k{$_}||0)."${xmlrs}".int($_url_e{$_}||0)."${xmlrs}".int($_url_x{$_}||0)."${xmlre}\n"; } foreach (keys %_url_p) { if ($keysinkeylist{$_}) { next; } my $newkey=$_; $newkey =~ s/([^:])\/\//$1\//g; # Because some targeted url were taped with 2 / (Ex: //rep//file.htm). We must keep http://rep/file.htm print HISTORYTMP "${xmlrb}$newkey ".int($_url_p{$_}||0)."${xmlrs}".int($_url_k{$_}||0)."${xmlrs}".int($_url_e{$_}||0)."${xmlrs}".int($_url_x{$_}||0)."${xmlre}\n"; } print HISTORYTMP "${xmleb}END_SIDER${xmlee}\n"; } if ($sectiontosave eq 'filetypes') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Files type - Hits - Bandwidth - Bandwidth without compression - Bandwidth after compression\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_FILETYPES${xmlbs}".(scalar keys %_filetypes_h)."${xmlbe}\n"; foreach (keys %_filetypes_h) { my $hits=$_filetypes_h{$_}||0; my $bytes=$_filetypes_k{$_}||0; my $bytesbefore=$_filetypes_gz_in{$_}||0; my $bytesafter=$_filetypes_gz_out{$_}||0; print HISTORYTMP "${xmlrb}$_${xmlrs}$hits${xmlrs}$bytes${xmlrs}$bytesbefore${xmlrs}$bytesafter${xmlre}\n"; } print HISTORYTMP "${xmleb}END_FILETYPES${xmlee}\n"; } if ($sectiontosave eq 'os') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# OS ID - Hits\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_OS${xmlbs}".(scalar keys %_os_h)."${xmlbe}\n"; foreach (keys %_os_h) { print HISTORYTMP "${xmlrb}$_${xmlrs}$_os_h{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_OS${xmlee}\n"; } if ($sectiontosave eq 'browser') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Browser ID - Hits\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_BROWSER${xmlbs}".(scalar keys %_browser_h)."${xmlbe}\n"; foreach (keys %_browser_h) { print HISTORYTMP "${xmlrb}$_${xmlrs}$_browser_h{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_BROWSER${xmlee}\n"; } if ($sectiontosave eq 'screensize') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Screen size - Hits\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_SCREENSIZE${xmlbs}".(scalar keys %_screensize_h)."${xmlbe}\n"; foreach (keys %_screensize_h) { print HISTORYTMP "${xmlrb}$_${xmlrs}$_screensize_h{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_SCREENSIZE${xmlee}\n"; } # Referer if ($sectiontosave eq 'unknownreferer') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Unknown referer OS - Last visit date\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_UNKNOWNREFERER${xmlbs}".(scalar keys %_unknownreferer_l)."${xmlbe}\n"; foreach (keys %_unknownreferer_l) { print HISTORYTMP "${xmlrb}$_${xmlrs}$_unknownreferer_l{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_UNKNOWNREFERER${xmlee}\n"; } if ($sectiontosave eq 'unknownrefererbrowser') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Unknown referer Browser - Last visit date\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_UNKNOWNREFERERBROWSER${xmlbs}".(scalar keys %_unknownrefererbrowser_l)."${xmlbe}\n"; foreach (keys %_unknownrefererbrowser_l) { print HISTORYTMP "${xmlrb}$_${xmlrs}$_unknownrefererbrowser_l{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_UNKNOWNREFERERBROWSER${xmlee}\n"; } if ($sectiontosave eq 'origin') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Origin - Pages - Hits \n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_ORIGIN${xmlbs}6"."${xmlbe}\n"; print HISTORYTMP "${xmlrb}From0${xmlrs}".int($_from_p[0])."${xmlrs}".int($_from_h[0])."${xmlre}\n"; print HISTORYTMP "${xmlrb}From1${xmlrs}".int($_from_p[1])."${xmlrs}".int($_from_h[1])."${xmlre}\n"; print HISTORYTMP "${xmlrb}From2${xmlrs}".int($_from_p[2])."${xmlrs}".int($_from_h[2])."${xmlre}\n"; print HISTORYTMP "${xmlrb}From3${xmlrs}".int($_from_p[3])."${xmlrs}".int($_from_h[3])."${xmlre}\n"; print HISTORYTMP "${xmlrb}From4${xmlrs}".int($_from_p[4])."${xmlrs}".int($_from_h[4])."${xmlre}\n"; # Same site print HISTORYTMP "${xmlrb}From5${xmlrs}".int($_from_p[5])."${xmlrs}".int($_from_h[5])."${xmlre}\n"; # News print HISTORYTMP "${xmleb}END_ORIGIN${xmlee}\n"; } if ($sectiontosave eq 'sereferrals') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Search engine referers ID - Pages - Hits\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_SEREFERRALS${xmlbs}".(scalar keys %_se_referrals_h)."${xmlbe}\n"; foreach (keys %_se_referrals_h) { print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_se_referrals_p{$_}||0)."${xmlrs}$_se_referrals_h{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_SEREFERRALS${xmlee}\n"; } if ($sectiontosave eq 'pagerefs') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'RefererShown'}\n"; } print HISTORYTMP "# External page referers - Pages - Hits\n"; print HISTORYTMP "# The $MaxNbOf{'RefererShown'} first Pages must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_PAGEREFS${xmlbs}".(scalar keys %_pagesrefs_h)."${xmlbe}\n"; # We save page list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'RefererShown'},$MinHit{'Refer'},\%_pagesrefs_h,\%_pagesrefs_p); %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; my $newkey=$_; $newkey =~ s/^http(s|):\/\/([^\/]+)\/$/http$1:\/\/$2/i; # Remove / at end of http://.../ but not at end of http://.../dir/ print HISTORYTMP "${xmlrb}".XMLEncodeForHisto($newkey)."${xmlrs}".int($_pagesrefs_p{$_}||0)."${xmlrs}$_pagesrefs_h{$_}${xmlre}\n"; } foreach (keys %_pagesrefs_h) { if ($keysinkeylist{$_}) { next; } my $newkey=$_; $newkey =~ s/^http(s|):\/\/([^\/]+)\/$/http$1:\/\/$2/i; # Remove / at end of http://.../ but not at end of http://.../dir/ print HISTORYTMP "${xmlrb}".XMLEncodeForHisto($newkey)."${xmlrs}".int($_pagesrefs_p{$_}||0)."${xmlrs}$_pagesrefs_h{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_PAGEREFS${xmlee}\n"; } if ($sectiontosave eq 'searchwords') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'KeyphrasesShown'}\n"; } print HISTORYTMP "# Search keyphrases - Number of search\n"; print HISTORYTMP "# The $MaxNbOf{'KeyphrasesShown'} first number of search must be first (order not required for others)\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_SEARCHWORDS${xmlbs}".(scalar keys %_keyphrases)."${xmlbe}\n"; # We will also build _keywords %_keywords=(); # We save key list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'KeywordsShown'},$MinHit{'Keyword'},\%_keyphrases,\%_keyphrases); %keysinkeylist=(); foreach my $key (@keylist) { $keysinkeylist{$key}=1; my $keyphrase=$key; print HISTORYTMP "${xmlrb}$keyphrase${xmlrs}$_keyphrases{$key}${xmlre}\n"; foreach (split(/\+/,$key)) { $_keywords{$_}+=$_keyphrases{$key}; } # To init %_keywords } foreach my $key (keys %_keyphrases) { if ($keysinkeylist{$key}) { next; } my $keyphrase=$key; print HISTORYTMP "${xmlrb}$keyphrase${xmlrs}$_keyphrases{$key}${xmlre}\n"; foreach (split(/\+/,$key)) { $_keywords{$_}+=$_keyphrases{$key}; } # To init %_keywords } print HISTORYTMP "${xmleb}END_SEARCHWORDS${xmlee}\n"; # Now save keywords section print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOf{'KeywordsShown'}\n"; } print HISTORYTMP "# Search keywords - Number of search\n"; print HISTORYTMP "# The $MaxNbOf{'KeywordsShown'} first number of search must be first (order not required for others)\n"; $ValueInFile{"keywords"}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_KEYWORDS${xmlbs}".(scalar keys %_keywords)."${xmlbe}\n"; # We save key list in score sorted order to get a -output faster and with less use of memory. &BuildKeyList($MaxNbOf{'KeywordsShown'},$MinHit{'Keyword'},\%_keywords,\%_keywords); %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; my $keyword=$_; print HISTORYTMP "${xmlrb}$keyword${xmlrs}$_keywords{$_}${xmlre}\n"; } foreach (keys %_keywords) { if ($keysinkeylist{$_}) { next; } my $keyword=$_; print HISTORYTMP "${xmlrb}$keyword${xmlrs}$_keywords{$_}${xmlre}\n"; } print HISTORYTMP "${xmleb}END_KEYWORDS${xmlee}\n"; } # Other - Errors if ($sectiontosave eq 'cluster') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Cluster ID - Pages - Hits - Bandwidth\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_CLUSTER${xmlbs}".(scalar keys %_cluster_h)."${xmlbe}\n"; foreach (keys %_cluster_h) { print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_cluster_p{$_}||0)."${xmlrs}".int($_cluster_h{$_}||0)."${xmlrs}".int($_cluster_k{$_}||0)."${xmlre}\n"; } print HISTORYTMP "${xmleb}END_CLUSTER${xmlee}\n"; } if ($sectiontosave eq 'misc') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Misc ID - Pages - Hits - Bandwidth\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_MISC${xmlbs}".(scalar keys %MiscListCalc)."${xmlbe}\n"; foreach (keys %MiscListCalc) { print HISTORYTMP "${xmlrb}$_${xmlrs}".int($_misc_p{$_}||0)."${xmlrs}".int($_misc_h{$_}||0)."${xmlrs}".int($_misc_k{$_}||0)."${xmlre}\n"; } print HISTORYTMP "${xmleb}END_MISC${xmlee}\n"; } if ($sectiontosave eq 'errors') { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# Errors - Hits - Bandwidth\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_ERRORS${xmlbs}".(scalar keys %_errors_h)."${xmlbe}\n"; foreach (keys %_errors_h) { print HISTORYTMP "${xmlrb}$_${xmlrs}$_errors_h{$_}${xmlrs}".int($_errors_k{$_}||0)."${xmlre}\n"; } print HISTORYTMP "${xmleb}END_ERRORS${xmlee}\n"; } # Other - Trapped errors foreach my $code (keys %TrapInfosForHTTPErrorCodes) { if ($sectiontosave eq "sider_$code") { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
\n"; } print HISTORYTMP "# URL with $code errors - Hits - Last URL referer\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_SIDER_$code${xmlbs}".(scalar keys %_sider404_h)."${xmlbe}\n"; foreach (keys %_sider404_h) { my $newkey=$_; my $newreferer=$_referer404_h{$_}||''; print HISTORYTMP "${xmlrb}".XMLEncodeForHisto($newkey)."${xmlrs}$_sider404_h{$_}${xmlrs}".XMLEncodeForHisto($newreferer)."${xmlre}\n"; } print HISTORYTMP "${xmleb}END_SIDER_$code${xmlee}\n"; } } # Other - Extra stats sections foreach my $extranum (1..@ExtraName-1) { if ($sectiontosave eq "extra_$extranum") { print HISTORYTMP "\n"; if ($xml) { print HISTORYTMP "
$MaxNbOfExtra[$extranum]\n"; } print HISTORYTMP "# Extra key - Pages - Hits - Bandwidth - Last access\n"; print HISTORYTMP "# The $MaxNbOfExtra[$extranum] first number of hits are first\n"; $ValueInFile{$sectiontosave}=tell HISTORYTMP; print HISTORYTMP "${xmlbb}BEGIN_EXTRA_$extranum${xmlbs}".scalar (keys %{'_section_' . $extranum . '_h'})."${xmlbe}\n"; &BuildKeyList($MaxNbOfExtra[$extranum],$MinHitExtra[$extranum],\%{'_section_' . $extranum . '_h'},\%{'_section_' . $extranum . '_p'}); %keysinkeylist=(); foreach (@keylist) { $keysinkeylist{$_}=1; my $page=${'_section_' . $extranum . '_p'}{$_}||0; my $bytes=${'_section_' . $extranum . '_k'}{$_}||0; my $lastaccess=${'_section_' . $extranum . '_l'}{$_}||''; print HISTORYTMP "${xmlrb}$_${xmlrs}$page${xmlrs}", ${'_section_' . $extranum . '_h'}{$_}, "${xmlrs}$bytes${xmlrs}$lastaccess${xmlre}\n"; next; } foreach (keys %{'_section_' . $extranum . '_h'}) { if ($keysinkeylist{$_}) { next; } my $page=${'_section_' . $extranum . '_p'}{$_}||0; my $bytes=${'_section_' . $extranum . '_k'}{$_}||0; my $lastaccess=${'_section_' . $extranum . '_l'}{$_}||''; print HISTORYTMP "${xmlrb}$_${xmlrs}$page${xmlrs}", ${'_section_' . $extranum . '_h'}{$_}, "${xmlrs}$bytes${xmlrs}$lastaccess${xmlre}\n"; next; } print HISTORYTMP "${xmleb}END_EXTRA_$extranum${xmlee}\n"; } } if ($sectiontosave eq "petitions") { if($condition == 0){ $condition = 1; if($position < 10){ if($counter > 0){ @request[$position] = $counter; @date_request[$position] = $lastdate; $position++; } } else{ if($position == 10){ $counterSmall = @request[0]; $positionSmall = 0; $i=0; foreach (@request){ if($counterSmall > @request[$i]){ $counterSmall = @request[$i]; $positionSmall = $i; } $i++; } if(@request[$positionSmall] < $counter){ @request[$positionSmall] = $counter; @date_request[$positionSmall] = $lastdate; } } } } $elements = $#request + 1; print HISTORYTMP "\n#Max Number of Petitions\n"; print HISTORYTMP "${xmleb}PETITION_BEGIN${xmlee} ".$elements."\n"; $element_copy = 0; for($k=0;$k<$elements-1;$k++){ $j = @request[$k]; for($i=$k+1;$i<$elements;$i++){ if($j < @request[$i]){ $element_copy = @request[$i]; #it set in orden array request @request[$i] = $j; @request[$k] = $element_copy; $j = @date_request[$k]; $element_copy = @date_request[$i]; #it set in orden array date_request @date_request[$i] = $j; @date_request[$k] = $element_copy; } $j = @request[$k]; } } $i = 0; $j = 0; foreach $i (@request){ print HISTORYTMP "@date_request[$j] $i\n"; $j++; } print HISTORYTMP "${xmleb}PETITION_END${xmlee}\n"; } # Other - Plugin sections if ($AtLeastOneSectionPlugin && $sectiontosave =~ /^plugin_(\w+)$/i) { my $pluginname=$1; if ($PluginsLoaded{'SectionInitHashArray'}{"$pluginname"}) { my $function="SectionWriteHistory_$pluginname(\$xml,\$xmlbb,\$xmlbs,\$xmlbe,\$xmlrb,\$xmlrs,\$xmlre,\$xmleb,\$xmlee)"; eval("$function"); last; } } %keysinkeylist=(); } #-------------------------------------------------------------------- # Function: Rename all tmp history file into history # Parameters: None # Input: $DirData $PROG $FileSuffix # $KeepBackupOfHistoricFile $SaveDatabaseFilesWithPermissionsForEveryone # Output: None # Return: 1 Ok, 0 at least one error (tmp files are removed) #-------------------------------------------------------------------- sub Rename_All_Tmp_History { my $pid=$$; my $renameok=1; if ($Debug) { debug("Call to Rename_All_Tmp_History (FileSuffix=$FileSuffix)"); } opendir(DIR,"$DirData"); my $regfilesuffix=quotemeta($FileSuffix); foreach (grep /^$PROG(\d\d\d\d\d\d)$regfilesuffix\.tmp\.$pid$/, sort readdir DIR) { /^$PROG(\d\d\d\d\d\d)$regfilesuffix\.tmp\.$pid$/; if ($renameok) { # No rename error yet if ($Debug) { debug(" Rename new tmp history file $PROG$1$FileSuffix.tmp.$$ into $PROG$1$FileSuffix.txt",1); } if (-s "$DirData/$PROG$1$FileSuffix.tmp.$$") { # Rename tmp files if size > 0 if ($KeepBackupOfHistoricFiles) { if (-s "$DirData/$PROG$1$FileSuffix.txt") { # History file already exists. We backup it if ($Debug) { debug(" Make a backup of old history file into $PROG$1$FileSuffix.bak before",1); } #if (FileCopy("$DirData/$PROG$1$FileSuffix.txt","$DirData/$PROG$1$FileSuffix.bak")) { if (rename("$DirData/$PROG$1$FileSuffix.txt", "$DirData/$PROG$1$FileSuffix.bak")==0) { warning("Warning: Failed to make a backup of \"$DirData/$PROG$1$FileSuffix.txt\" into \"$DirData/$PROG$1$FileSuffix.bak\"."); } if ($SaveDatabaseFilesWithPermissionsForEveryone) { chmod 0666,"$DirData/$PROG$1$FileSuffix.bak"; } } else { if ($Debug) { debug(" No need to backup old history file",1); } } } if (rename("$DirData/$PROG$1$FileSuffix.tmp.$$", "$DirData/$PROG$1$FileSuffix.txt")==0) { $renameok=0; # At least one error in renaming working files # Remove tmp file unlink "$DirData/$PROG$1$FileSuffix.tmp.$$"; warning("Warning: Failed to rename \"$DirData/$PROG$1$FileSuffix.tmp.$$\" into \"$DirData/$PROG$1$FileSuffix.txt\".\nWrite permissions on \"$PROG$1$FileSuffix.txt\" might be wrong".($ENV{'GATEWAY_INTERFACE'}?" for an 'update from web'":"")." or file might be opened."); next; } if ($SaveDatabaseFilesWithPermissionsForEveryone) { chmod 0666,"$DirData/$PROG$1$FileSuffix.txt"; } } } else { # Because of rename error, we remove all remaining tmp files unlink "$DirData/$PROG$1$FileSuffix.tmp.$$"; } } close DIR; return $renameok; } #------------------------------------------------------------------------------ # Function: Load DNS cache file entries into a memory hash array # Parameters: Hash array ref to load into, # File name to load, # File suffix to use # Save to a second plugin file if not up to date # Input: None # Output: Hash array is loaded # Return: 1 No DNS Cache file found, 0 OK #------------------------------------------------------------------------------ sub Read_DNS_Cache { my $hashtoload=shift; my $dnscachefile=shift; my $filesuffix=shift; my $savetohash=shift; my $dnscacheext=''; my $filetoload=''; my $timetoload = time(); if ($Debug) { debug("Call to Read_DNS_Cache [file=\"$dnscachefile\"]"); } if ($dnscachefile =~ s/(\.\w+)$//) { $dnscacheext=$1; } foreach my $dir ("$DirData",".","") { my $searchdir=$dir; if ($searchdir && (!($searchdir =~ /\/$/)) && (!($searchdir =~ /\\$/)) ) { $searchdir .= "/"; } if (-f "${searchdir}$dnscachefile$filesuffix$dnscacheext") { $filetoload="${searchdir}$dnscachefile$filesuffix$dnscacheext"; } # Plugin call : Change filetoload if ($PluginsLoaded{'SearchFile'}{'hashfiles'}) { SearchFile_hashfiles($searchdir,$dnscachefile,$filesuffix,$dnscacheext,$filetoload); } if ($filetoload) { last; } # We found a file to load } if (! $filetoload) { if ($Debug) { debug(" No DNS Cache file found"); } return 1; } # Plugin call : Load hashtoload if ($PluginsLoaded{'LoadCache'}{'hashfiles'}) { LoadCache_hashfiles($filetoload,$hashtoload); } if (! scalar keys %$hashtoload) { open(DNSFILE,"$filetoload") or error("Couldn't open DNS Cache file \"$filetoload\": $!"); #binmode DNSFILE; # If we set binmode here, it seems that the load is broken on ActiveState 5.8 # This is a fast way to load with regexp %$hashtoload = map(/^(?:\d{0,10}\s+)?([0-9A-F:\.]+)\s+([^\s]+)$/oi,); close DNSFILE; if ($savetohash) { # Plugin call : Save hash file (all records) with test if up to date to save if ($PluginsLoaded{'SaveHash'}{'hashfiles'}) { SaveHash_hashfiles($filetoload,$hashtoload,1,0); } } } if ($Debug) { debug(" Loaded ".(scalar keys %$hashtoload)." items from $filetoload in ".(time()-$timetoload)." seconds.",1); } return 0; } #------------------------------------------------------------------------------ # Function: Save a memory hash array into a DNS cache file # Parameters: Hash array ref to save, # File name to save, # File suffix to use # Input: None # Output: None # Return: 0 OK, 1 Error #------------------------------------------------------------------------------ sub Save_DNS_Cache_File { my $hashtosave=shift; my $dnscachefile=shift; my $filesuffix=shift; my $dnscacheext=''; my $filetosave=''; my $timetosave = time(); my $nbofelemtosave=$NBOFLASTUPDATELOOKUPTOSAVE; my $nbofelemsaved=0; if ($Debug) { debug("Call to Save_DNS_Cache_File [file=\"$dnscachefile\"]"); } if (! scalar keys %$hashtosave) { if ($Debug) { debug(" No data to save"); } return 0; } if ($dnscachefile =~ s/(\.\w+)$//) { $dnscacheext=$1; } $filetosave="$dnscachefile$filesuffix$dnscacheext"; # Plugin call : Save hash file (only $NBOFLASTUPDATELOOKUPTOSAVE records) with no test if up to date if ($PluginsLoaded{'SaveHash'}{'hashfiles'}) { SaveHash_hashfiles($filetosave,$hashtosave,0,$nbofelemtosave,$nbofelemsaved); } if (! $nbofelemsaved) { $filetosave="$dnscachefile$filesuffix$dnscacheext"; if ($Debug) { debug(" Save data ".($nbofelemtosave?"($nbofelemtosave records max)":"(all records)")." into file $filetosave"); } if (! open(DNSFILE,">$filetosave")) { warning("Warning: Failed to open for writing last update DNS Cache file \"$filetosave\": $!"); return 1; } binmode DNSFILE; my $starttimemin=int($starttime/60); foreach my $key (keys %$hashtosave) { #if ($hashtosave->{$key} ne '*') { my $ipsolved=$hashtosave->{$key}; print DNSFILE "$starttimemin\t$key\t".($ipsolved eq 'ip'?'*':$ipsolved)."\n"; # Change 'ip' to '*' for backward compatibility if (++$nbofelemsaved >= $NBOFLASTUPDATELOOKUPTOSAVE) { last; } #} } close DNSFILE; if ($SaveDatabaseFilesWithPermissionsForEveryone) { chmod 0666,"$filetosave"; } } if ($Debug) { debug(" Saved $nbofelemsaved items into $filetosave in ".(time()-$timetosave)." seconds.",1); } return 0; } #------------------------------------------------------------------------------ # Function: Return time elapsed since last call in miliseconds # Parameters: 0|1 (0 reset counter, 1 no reset) # Input: None # Output: None # Return: Number of miliseconds elapsed since last call #------------------------------------------------------------------------------ sub GetDelaySinceStart { if (shift) { $StartSeconds=0; } # Reset chrono my ($newseconds, $newmicroseconds)=(time(),0); # Plugin call : Return seconds and milliseconds if ($PluginsLoaded{'GetTime'}{'timehires'}) { GetTime_timehires($newseconds, $newmicroseconds); } if (! $StartSeconds) { $StartSeconds=$newseconds; $StartMicroseconds=$newmicroseconds; } return (($newseconds-$StartSeconds)*1000+int(($newmicroseconds-$StartMicroseconds)/1000)); } #------------------------------------------------------------------------------ # Function: Reset all variables whose name start with _ because a new month start # Parameters: None # Input: $YearRequired All variables whose name start with _ # Output: All variables whose name start with _ # Return: None #------------------------------------------------------------------------------ sub Init_HashArray { if ($Debug) { debug("Call to Init_HashArray"); } # Reset global hash arrays %FirstTime = %LastTime = (); %MonthHostsKnown = %MonthHostsUnknown = (); %MonthVisits = %MonthUnique = (); %MonthPages = %MonthHits = %MonthBytes = (); %MonthNotViewedPages = %MonthNotViewedHits = %MonthNotViewedBytes = (); %DayPages = %DayHits = %DayBytes = %DayVisits = (); # Reset all arrays with name beginning by _ for (my $ix=0; $ix<6; $ix++) { $_from_p[$ix]=0; $_from_h[$ix]=0; } for (my $ix=0; $ix<24; $ix++) { $_time_h[$ix]=0; $_time_k[$ix]=0; $_time_p[$ix]=0; $_time_nv_h[$ix]=0; $_time_nv_k[$ix]=0; $_time_nv_p[$ix]=0; } # Reset all hash arrays with name beginning by _ %_session = %_browser_h = (); %_domener_p = %_domener_h = %_domener_k = %_errors_h = %_errors_k = (); %_filetypes_h = %_filetypes_k = %_filetypes_gz_in = %_filetypes_gz_out = (); %_host_p = %_host_h = %_host_k = %_host_l = %_host_s = %_host_u = (); %_waithost_e = %_waithost_l = %_waithost_s = %_waithost_u = (); %_keyphrases = %_keywords = %_os_h = %_pagesrefs_p = %_pagesrefs_h = %_robot_h = %_robot_k = %_robot_l = %_robot_r = (); %_worm_h = %_worm_k = %_worm_l = %_login_p = %_login_h = %_login_k = %_login_l = %_screensize_h = (); %_misc_p = %_misc_h = %_misc_k = (); %_cluster_p = %_cluster_h = %_cluster_k = (); %_se_referrals_p = %_se_referrals_h = %_sider404_h = %_referer404_h = %_url_p = %_url_k = %_url_e = %_url_x = (); %_unknownreferer_l = %_unknownrefererbrowser_l = (); %_emails_h = %_emails_k = %_emails_l = %_emailr_h = %_emailr_k = %_emailr_l = (); for (my $ix=1; $ix < @ExtraName; $ix++) { %{'_section_' . $ix . '_h'} = %{'_section_' . $ix . '_o'} = %{'_section_' . $ix . '_k'} = %{'_section_' . $ix . '_l'} = %{'_section_' . $ix . '_p'} = (); } foreach my $pluginname (keys %{$PluginsLoaded{'SectionInitHashArray'}}) { my $function="SectionInitHashArray_$pluginname()"; eval("$function"); } } #------------------------------------------------------------------------------ # Function: Change word separators of a keyphrase string into space and # remove bad coded chars # Parameters: stringtodecode # Input: None # Output: None # Return: decodedstring #------------------------------------------------------------------------------ sub ChangeWordSeparatorsIntoSpace { $_[0] =~ s/%0[ad]/ /ig; # LF CR $_[0] =~ s/%2[02789abc]/ /ig; # space " ' ( ) * + , $_[0] =~ s/%3a/ /ig; # : $_[0] =~ tr/\+\'\(\)\"\*,:/ /s; # "&" and "=" must not be in this list } #------------------------------------------------------------------------------ # Function: Transforms & into & as needed in XML/XHTML # Parameters: stringtoencode # Return: encodedstring #------------------------------------------------------------------------------ sub XMLEncode { if ($BuildReportFormat ne 'xhtml' && $BuildReportFormat ne 'xml') { return shift; } my $string = shift; $string =~ s/&/&/g; return $string; } #------------------------------------------------------------------------------ # Function: Transforms & into & as needed in XML/XHTML # Parameters: stringtoencode # Return: encodedstring #------------------------------------------------------------------------------ sub XMLEncodeForHisto { my $string = shift; $string =~ s/\s/%20/g; if ($BuildHistoryFormat ne 'xml') { return $string; } $string =~ s/&/&/g; $string =~ s//>/g; return $string; } #------------------------------------------------------------------------------ # Function: Encode a binary string into an ASCII string # Parameters: stringtoencode # Return: encodedstring #------------------------------------------------------------------------------ sub EncodeString { my $string = shift; # use bytes; $string =~ s/([\x2B\x80-\xFF])/sprintf ("%%%2x", ord($1))/eg; # no bytes; $string =~ tr/ /+/s; return $string; } #------------------------------------------------------------------------------ # Function: Decode an only text string into a binary string # Parameters: stringtodecode # Input: None # Output: None # Return: decodedstring #------------------------------------------------------------------------------ sub DecodeEncodedString { my $stringtodecode=shift; $stringtodecode =~ tr/\+/ /s; $stringtodecode =~ s/%([A-F0-9][A-F0-9])/pack("C", hex($1))/ieg; return $stringtodecode; } #------------------------------------------------------------------------------ # Function: Decode an precompiled regex value to a common regex value # Parameters: compiledregextodecode # Input: None # Output: None # Return: standardregex #------------------------------------------------------------------------------ sub UnCompileRegex { shift =~ /\(\?[-\w]*:(.*)\)/; return $1; } #------------------------------------------------------------------------------ # Function: Clean a string of HTML tags to avoid 'Cross Site Scripting attacks' # Parameters: stringtodecode # Input: None # Output: None # Return: decodedstring #------------------------------------------------------------------------------ sub CleanFromCSSA { my $stringtoclean=shift; $stringtoclean =~ s//>/g; return $stringtoclean; } #------------------------------------------------------------------------------ # Function: Clean tags in a string # Parameters: stringtodecode # Input: None # Output: None # Return: decodedstring #------------------------------------------------------------------------------ sub CleanFromTags { my $stringtoclean=shift; $stringtoclean =~ s/$regclean1/ /g; # Replace or with space $stringtoclean =~ s/$regclean2//g; # Remove return $stringtoclean; } #------------------------------------------------------------------------------ # Function: Copy one file into another # Parameters: sourcefilename targetfilename # Input: None # Output: None # Return: 0 if copy is ok, 1 else #------------------------------------------------------------------------------ sub FileCopy { my $filesource = shift; my $filetarget = shift; if ($Debug) { debug("FileCopy($filesource,$filetarget)",1); } open(FILESOURCE,"$filesource") || return 1; open(FILETARGET,">$filetarget") || return 1; binmode FILESOURCE; binmode FILETARGET; # ... close(FILETARGET); close(FILESOURCE); if ($Debug) { debug(" File copied",1); } return 0; } #------------------------------------------------------------------------------ # Function: Show flags for other language translations # Parameters: Current languade id (en, fr, ...) # Input: None # Output: None # Return: None #------------------------------------------------------------------------------ sub Show_Flag_Links { my $CurrentLang = shift; # Build flags link my $NewLinkParams=$QueryString; my $NewLinkTarget=''; if ($ENV{'GATEWAY_INTERFACE'}) { $NewLinkParams =~ s/(^|&)update(=\w*|$)//i; $NewLinkParams =~ s/(^|&)staticlinks(=\w*|$)//i; $NewLinkParams =~ s/(^|&)framename=[^&]*//i; $NewLinkParams =~ s/(^|&)lang=[^&]*//i; if ($FrameName eq 'mainright') { $NewLinkTarget=" target=\"_parent\""; } $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; if ($NewLinkParams) { $NewLinkParams="${NewLinkParams}&"; } } else { $NewLinkParams=($SiteConfig?"config=$SiteConfig&":"")."year=$YearRequired&month=$MonthRequired&"; } if ($FrameName eq 'mainright') { $NewLinkParams.='framename=index&'; } foreach my $lng (split(/\s+/,$ShowFlagLinks)) { $lng=$LangBrowserToLangAwstats{$lng}?$LangBrowserToLangAwstats{$lng}:$lng; if ($lng ne $CurrentLang) { my %lngtitle=('en','English','fr','French','de','German','it','Italian','nl','Dutch','es','Spanish'); my $lngtitle=($lngtitle{$lng}?$lngtitle{$lng}:$lng); my $flag=($LangAWStatsToFlagAwstats{$lng}?$LangAWStatsToFlagAwstats{$lng}:$lng); print " \n"; } } } #------------------------------------------------------------------------------ # Function: Format value in bytes in a string (Bytes, Kb, Mb, Gb) # Parameters: bytes (integer value or "0.00") # Input: None # Output: None # Return: "x.yz MB" or "x.yy KB" or "x Bytes" or "0" #------------------------------------------------------------------------------ sub Format_Bytes { my $bytes = shift||0; my $fudge = 1; # Do not use exp/log function to calculate 1024power, function make segfault on some unix/perl versions if ($bytes >= ($fudge << 30)) { return sprintf("%.2f", $bytes/1073741824)." $Message[110]"; } if ($bytes >= ($fudge << 20)) { return sprintf("%.2f", $bytes/1048576)." $Message[109]"; } if ($bytes >= ($fudge << 10)) { return sprintf("%.2f", $bytes/1024)." $Message[108]"; } if ($bytes < 0) { $bytes="?"; } return int($bytes).(int($bytes)?" $Message[119]":""); } #------------------------------------------------------------------------------ # Function: Return " alt=string title=string" # Parameters: string # Input: None # Output: None # Return: "alt=string title=string" #------------------------------------------------------------------------------ sub AltTitle { my $string = shift||''; return " alt='$string' title='$string'"; # return " alt=\"$string\" title=\"$string\""; # return ($BuildReportFormat?"":" alt=\"$string\"")." title=\"$string\""; } #------------------------------------------------------------------------------ # Function: Tell if an email is a local or external email # Parameters: email # Input: $SiteDomain(exact string) $HostAliases(quoted regex string) # Output: None # Return: -1, 0 or 1 #------------------------------------------------------------------------------ sub IsLocalEMail { my $email=shift||'unknown'; if ($email !~ /\@(.*)$/) { return 0; } my $domain=$1; if ($domain =~ /^$SiteDomain$/i) { return 1; } foreach (@HostAliases) { if ($domain =~ /$_/) { return 1; } } return -1; } #------------------------------------------------------------------------------ # Function: Format a date according to Message[78] (country date format) # Parameters: String date YYYYMMDDHHMMSS # Option 0=LastUpdate and LastTime date # 1=Arrays date except daymonthvalues # 2=daymonthvalues date (only year month and day) # Input: $Message[78] # Output: None # Return: Date with format defined by Message[78] and option #------------------------------------------------------------------------------ sub Format_Date { my $date=shift; my $option=shift||0; my $year=substr("$date",0,4); my $month=substr("$date",4,2); my $day=substr("$date",6,2); my $hour=substr("$date",8,2); my $min=substr("$date",10,2); my $sec=substr("$date",12,2); my $dateformat=$Message[78]; if ($option == 2) { $dateformat =~ s/^[^ymd]+//g; $dateformat =~ s/[^ymd]+$//g; } $dateformat =~ s/yyyy/$year/g; $dateformat =~ s/yy/$year/g; $dateformat =~ s/mmm/$MonthNumLib{$month}/g; $dateformat =~ s/mm/$month/g; $dateformat =~ s/dd/$day/g; $dateformat =~ s/HH/$hour/g; $dateformat =~ s/MM/$min/g; $dateformat =~ s/SS/$sec/g; return "$dateformat"; } #------------------------------------------------------------------------------ # Function: Return 1 if string contains only ascii chars # Parameters: string # Input: None # Output: None # Return: 0 or 1 #------------------------------------------------------------------------------ sub IsAscii { my $string=shift; if ($Debug) { debug("IsAscii($string)",5); } if ($string =~ /^[\w\+\-\/\\\.%,;:=\"\'&?!\s]+$/) { if ($Debug) { debug(" Yes",6); } return 1; # Only alphanum chars (and _) or + - / \ . % , ; : = " ' & ? space \t } if ($Debug) { debug(" No",6); } return 0; } #------------------------------------------------------------------------------ # Function: Return the lower value between 2 but exclude value if 0 # Parameters: Val1 and Val2 # Input: None # Output: None # Return: min(Val1,Val2) #------------------------------------------------------------------------------ sub MinimumButNoZero { my ($val1,$val2)=@_; return ($val1&&($val1<$val2||!$val2)?$val1:$val2); } #------------------------------------------------------------------------------ # Function: Add a val from sorting tree # Parameters: keytoadd keyval [firstadd] # Input: None # Output: None # Return: None #------------------------------------------------------------------------------ sub AddInTree { my $keytoadd=shift; my $keyval=shift; my $firstadd=shift||0; if ($firstadd==1) { # Val is the first one if ($Debug) { debug(" firstadd",4); } $val{$keyval}=$keytoadd; $lowerval=$keyval; if ($Debug) { debug(" lowerval=$lowerval, nb elem val=".(scalar keys %val).", nb elem egal=".(scalar keys %egal).".",4); } return; } if ($val{$keyval}) { # Val is already in tree if ($Debug) { debug(" val is already in tree",4); } $egal{$keytoadd}=$val{$keyval}; $val{$keyval}=$keytoadd; if ($Debug) { debug(" lowerval=$lowerval, nb elem val=".(scalar keys %val).", nb elem egal=".(scalar keys %egal).".",4); } return; } if ($keyval <= $lowerval) { # Val is a new one lower (should happens only when tree is not full) if ($Debug) { debug(" keytoadd val=$keyval is lower or equal to lowerval=$lowerval",4); } $val{$keyval}=$keytoadd; $nextval{$keyval}=$lowerval; $lowerval=$keyval; if ($Debug) { debug(" lowerval=$lowerval, nb elem val=".(scalar keys %val).", nb elem egal=".(scalar keys %egal).".",4); } return; } # Val is a new one higher if ($Debug) { debug(" keytoadd val=$keyval is higher than lowerval=$lowerval",4); } $val{$keyval}=$keytoadd; my $valcursor=$lowerval; # valcursor is value just before keyval while ($nextval{$valcursor} && ($nextval{$valcursor} < $keyval)) { $valcursor=$nextval{$valcursor}; } if ($nextval{$valcursor}) { # keyval is between valcursor and nextval{valcursor} $nextval{$keyval}=$nextval{$valcursor}; } $nextval{$valcursor}=$keyval; if ($Debug) { debug(" lowerval=$lowerval, nb elem val=".(scalar keys %val).", nb elem egal=".(scalar keys %egal).".",4); } } #------------------------------------------------------------------------------ # Function: Remove a val from sorting tree # Parameters: None # Input: $lowerval %val %egal # Output: None # Return: None #------------------------------------------------------------------------------ sub Removelowerval { my $keytoremove=$val{$lowerval}; # This is lower key if ($Debug) { debug(" remove for lowerval=$lowerval: key=$keytoremove",4); } if ($egal{$keytoremove}) { $val{$lowerval}=$egal{$keytoremove}; delete $egal{$keytoremove}; } else { delete $val{$lowerval}; $lowerval=$nextval{$lowerval}; # Set new lowerval } if ($Debug) { debug(" new lower value=$lowerval, val size=".(scalar keys %val).", egal size=".(scalar keys %egal),4); } } #------------------------------------------------------------------------------ # Function: Build @keylist array # Parameters: Size max for @keylist array, # Min value in hash for select, # Hash used for select, # Hash used for order # Input: None # Output: None # Return: @keylist response array #------------------------------------------------------------------------------ sub BuildKeyList { my $ArraySize=shift||error("System error. Call to BuildKeyList function with incorrect value for first param","","",1); my $MinValue=shift||error("System error. Call to BuildKeyList function with incorrect value for second param","","",1); my $hashforselect=shift; my $hashfororder=shift; if ($Debug) { debug(" BuildKeyList($ArraySize,$MinValue,$hashforselect with size=".(scalar keys %$hashforselect).",$hashfororder with size=".(scalar keys %$hashfororder).")",3); } delete $hashforselect->{0};delete $hashforselect->{''}; # Those is to protect from infinite loop when hash array has an incorrect null key my $count=0; $lowerval=0; # Global because used in AddInTree and Removelowerval %val=(); %nextval=(); %egal=(); foreach my $key (keys %$hashforselect) { if ($count < $ArraySize) { if ($hashforselect->{$key} >= $MinValue) { $count++; if ($Debug) { debug(" Add in tree entry $count : $key (value=".($hashfororder->{$key}||0).", tree not full)",4); } AddInTree($key,$hashfororder->{$key}||0,$count); } next; } $count++; if (($hashfororder->{$key}||0)<=$lowerval) { next; } if ($Debug) { debug(" Add in tree entry $count : $key (value=".($hashfororder->{$key}||0)." > lowerval=$lowerval)",4); } AddInTree($key,$hashfororder->{$key}||0); if ($Debug) { debug(" Removelower in tree",4); } Removelowerval(); } # Build key list and sort it if ($Debug) { debug(" Build key list and sort it. lowerval=$lowerval, nb elem val=".(scalar keys %val).", nb elem egal=".(scalar keys %egal).".",3); } my %notsortedkeylist=(); foreach my $key (values %val) { $notsortedkeylist{$key}=1; } foreach my $key (values %egal) { $notsortedkeylist{$key}=1; } @keylist=(); @keylist=(sort {($hashfororder->{$b}||0) <=> ($hashfororder->{$a}||0) } keys %notsortedkeylist); if ($Debug) { debug(" BuildKeyList End (keylist size=".(@keylist).")",3); } return; } #------------------------------------------------------------------------------ # Function: Lock or unlock update # Parameters: status (1 to lock, 0 to unlock) # Input: $DirLock (if status=0) $PROG $FileSuffix # Output: $DirLock (if status=1) # Return: None #------------------------------------------------------------------------------ sub Lock_Update { my $status=shift; my $lock="$PROG$FileSuffix.lock"; if ($status) { # We stop if there is at least one lock file wherever it is foreach my $key ($ENV{"TEMP"},$ENV{"TMP"},"/tmp","/",".") { my $newkey =$key; $newkey =~ s/[\\\/]$//; if (-f "$newkey/$lock") { error("An AWStats update process seems to be already running for this config file. Try later.\nIf this is not true, remove manually lock file '$newkey/$lock'.","","",1); } } # Set lock where we can foreach my $key ($ENV{"TEMP"},$ENV{"TMP"},"/tmp","/",".") { if (! -d "$key") { next; } $DirLock=$key; $DirLock =~ s/[\\\/]$//; if ($Debug) { debug("Update lock file $DirLock/$lock is set"); } open(LOCK,">$DirLock/$lock") || error("Failed to create lock file $DirLock/$lock","","",1); print LOCK "AWStats update started by process $$ at $nowyear-$nowmonth-$nowday $nowhour:$nowmin:$nowsec\n"; close(LOCK); last; } } else { # Remove lock if ($Debug) { debug("Update lock file $DirLock/$lock is removed"); } unlink("$DirLock/$lock"); } return; } #------------------------------------------------------------------------------ # Function: Signal handler to call Lock_Update to remove lock file # Parameters: Signal name # Input: None # Output: None # Return: None #------------------------------------------------------------------------------ sub SigHandler { my $signame = shift; print ucfirst($PROG)." process (ID $$) interrupted by signal $signame.\n"; &Lock_Update(0); exit 1; } #------------------------------------------------------------------------------ # Function: Convert an IPAddress into an integer # Parameters: IPAddress # Input: None # Output: None # Return: Int #------------------------------------------------------------------------------ sub Convert_IP_To_Decimal { my ($IPAddress) = @_; my @ip_seg_arr = split(/\./,$IPAddress); my $decimal_ip_address = 256 * 256 *256 * $ip_seg_arr[0] + 256 * 256 * $ip_seg_arr[1] + 256 * $ip_seg_arr[2] + $ip_seg_arr[3]; return($decimal_ip_address); } #------------------------------------------------------------------------------ # Function: Test there is at least on value in list not null # Parameters: List of values # Input: None # Output: None # Return: 1 There is at least one not null value, 0 else #------------------------------------------------------------------------------ sub AtLeastOneNotNull { if ($Debug) { debug(" Call to AtLeastOneNotNull (".join('-',@_).")",3); } foreach my $val (@_) { if ($val) { return 1; } } return 0; } #------------------------------------------------------------------------------ # Function: Return the string to add in html tag to include popup javascript code # Parameters: tooltip number # Input: None # Output: None # Return: string with javascript code #------------------------------------------------------------------------------ sub Tooltip { my $ttnb=shift; return ($TOOLTIPON?" onmouseover=\"ShowTip($ttnb);\" onmouseout=\"HideTip($ttnb);\"":""); } #------------------------------------------------------------------------------ # Function: Insert a form filter # Parameters: Name of filter field, default for filter field, default for exclude filter field # Input: $StaticLinks, $QueryString, $SiteConfig, $DirConfig # Output: HTML Form # Return: None #------------------------------------------------------------------------------ sub ShowFormFilter { my $fieldfiltername=shift; my $fieldfilterinvalue=shift; my $fieldfilterexvalue=shift; if (! $StaticLinks) { my $NewLinkParams=${QueryString}; $NewLinkParams =~ s/(^|&)update(=\w*|$)//i; $NewLinkParams =~ s/(^|&)output(=\w*|$)//i; $NewLinkParams =~ s/(^|&)staticlinks(=\w*|$)//i; $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; if ($NewLinkParams) { $NewLinkParams="${NewLinkParams}&"; } print "\n
\n"; print "\n"; print "\n"; print "\n"; print ""; print "\n"; print "\n"; print "\n"; print ""; print "
$Message[79] :   $Message[153] :"; print "\n"; if ($SiteConfig) { print "\n"; } if ($DirConfig) { print "\n"; } if (@OnlyUsers) { print "\n"; } if (@OnlyLines) { print "\n"; } if ($SiteDomain) { print "\n"; } if ($QueryString =~ /(^|&)year=(\d\d\d\d)/i) { print "\n"; } if ($QueryString =~ /(^|&)month=(\d\d)/i || $QueryString =~ /(^|&)month=(all)/i) { print "\n"; } if ($QueryString =~ /(^|&)lang=(\w+)/i) { print "\n"; } if ($QueryString =~ /(^|&)debug=(\d+)/i) { print "\n"; } if ($QueryString =~ /(^|&)framename=(\w+)/i) { print "\n"; } print "  
\n"; print "
\n"; print "
\n"; print "\n"; } } #------------------------------------------------------------------------------ # Function: Write other user info (with help of plugin) # Parameters: $user # Input: $SiteConfig # Output: URL link # Return: None #------------------------------------------------------------------------------ sub ShowUserInfo { my $user=shift; # Call to plugins' function ShowInfoUser foreach my $pluginname (keys %{$PluginsLoaded{'ShowInfoUser'}}) { my $function="ShowInfoUser_$pluginname('$user')"; eval("$function"); } } #------------------------------------------------------------------------------ # Function: Write other cluster info (with help of plugin) # Parameters: $clusternb # Input: $SiteConfig # Output: Cluster info # Return: None #------------------------------------------------------------------------------ sub ShowClusterInfo { my $user=shift; # Call to plugins' function ShowInfoCluster foreach my $pluginname (keys %{$PluginsLoaded{'ShowInfoCluster'}}) { my $function="ShowInfoCluster_$pluginname('$user')"; eval("$function"); } } #------------------------------------------------------------------------------ # Function: Write other host info (with help of plugin) # Parameters: $host # Input: $LinksToWhoIs $LinksToWhoIsIp # Output: None # Return: None #------------------------------------------------------------------------------ sub ShowHostInfo { my $host=shift; # Call to plugins' function ShowInfoHost foreach my $pluginname (keys %{$PluginsLoaded{'ShowInfoHost'}}) { my $function="ShowInfoHost_$pluginname('$host')"; eval("$function"); } } #------------------------------------------------------------------------------ # Function: Write other url info (with help of plugin) # Parameters: $url # Input: %Aliases $MaxLengthOfShownURL $ShowLinksOnUrl $SiteDomain $UseHTTPSLinkForUrl # Output: URL link # Return: None #------------------------------------------------------------------------------ sub ShowURLInfo { my $url=shift; my $nompage=CleanFromCSSA($url); # Call to plugins' function ShowInfoURL foreach my $pluginname (keys %{$PluginsLoaded{'ShowInfoURL'}}) { my $function="ShowInfoURL_$pluginname('$url')"; eval("$function"); } if (length($nompage)>$MaxLengthOfShownURL) { $nompage=substr($nompage,0,$MaxLengthOfShownURL)."..."; } if ($ShowLinksOnUrl) { my $newkey=CleanFromCSSA($url); if ($LogType eq 'W' || $LogType eq 'S') { # Web or streaming log file if ($newkey =~ /^http(s|):/i) { # URL seems to be extracted from a proxy log file print "".XMLEncode($nompage).""; } elsif ($newkey =~ /^\//) { # URL seems to be an url extracted from a web or wap server log file $newkey =~ s/^\/$SiteDomain//i; # Define urlprot my $urlprot='http'; if ($UseHTTPSLinkForUrl && $newkey =~ /^$UseHTTPSLinkForUrl/) { $urlprot='https'; } print "".XMLEncode($nompage).""; } else { print XMLEncode($nompage); } } elsif ($LogType eq 'F') { # Ftp log file print XMLEncode($nompage); } elsif ($LogType eq 'M') { # Smtp log file print XMLEncode($nompage); } else { # Other type log file print XMLEncode($nompage); } } else { print XMLEncode($nompage); } } #------------------------------------------------------------------------------ # Function: Define value for PerlParsingFormat (used for regex log record parsing) # Parameters: - # Input: $LogFormat # Output: @fieldlib # Return: - #------------------------------------------------------------------------------ sub DefinePerlParsingFormat { # Log records examples: # Apache combined: 62.161.78.73 user - [dd/mmm/yyyy:hh:mm:ss +0000] "GET / HTTP/1.1" 200 1234 "http://www.from.com/from.htm" "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)" # Apache combined (408 error): my.domain.com - user [09/Jan/2001:11:38:51 -0600] "OPTIONS /mime-tmp/xxx file.doc HTTP/1.1" 408 - "-" "-" # Apache combined (408 error): 62.161.78.73 user - [dd/mmm/yyyy:hh:mm:ss +0000] "-" 408 - "-" "-" # Apache common_with_mod_gzip_info1: %h %l %u %t \"%r\" %>s %b mod_gzip: %{mod_gzip_compression_ratio}npct. # Apache common_with_mod_gzip_info2: %h %l %u %t \"%r\" %>s %b mod_gzip: %{mod_gzip_result}n In:%{mod_gzip_input_size}n Out:%{mod_gzip_output_size}n:%{mod_gzip_compression_ratio}npct. # Apache deflate: %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" (%{ratio}n) # IIS: 2000-07-19 14:14:14 62.161.78.73 - GET / 200 1234 HTTP/1.1 Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+NT+5.0) http://www.from.com/from.htm # WebStar: 05/21/00 00:17:31 OK 200 212.242.30.6 Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt) http://www.cover.dk/ "www.cover.dk" :Documentation:graphics:starninelogo.white.gif 1133 # Squid extended: 12.229.91.170 - - [27/Jun/2002:03:30:50 -0700] "GET http://www.callistocms.com/images/printable.gif HTTP/1.1" 304 354 "-" "Mozilla/5.0 Galeon/1.0.3 (X11; Linux i686; U;) Gecko/0" TCP_REFRESH_HIT:DIRECT if ($Debug) { debug("Call To DefinePerlParsingFormat (LogType='$LogType', LogFormat='$LogFormat')"); } @fieldlib=(); if ($LogFormat =~ /^[1-6]$/) { # Pre-defined log format if ($LogFormat eq '1' || $LogFormat eq '6') { # Same than "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"". # %u (user) is "([^\\[]+)" instead of "[^ ]+" because can contain space (Lotus Notes). referer and ua might be "". # $PerlParsingFormat="([^ ]+) [^ ]+ ([^\\[]+) \\[([^ ]+) [^ ]+\\] \\\"([^ ]+) (.+) [^\\\"]+\\\" ([\\d|-]+) ([\\d|-]+) \\\"(.*?)\\\" \\\"([^\\\"]*)\\\""; $PerlParsingFormat="([^ ]+) [^ ]+ ([^\\[]+) \\[([^ ]+) [^ ]+\\] \\\"([^ ]+) ([^ ]+) [^\\\"]+\\\" ([\\d|-]+) ([\\d|-]+) \\\"(.*?)\\\" \\\"([^\\\"]*)\\\""; $pos_host=0;$pos_logname=1;$pos_date=2;$pos_method=3;$pos_url=4;$pos_code=5;$pos_size=6;$pos_referer=7;$pos_agent=8; @fieldlib=('host','logname','date','method','url','code','size','referer','ua'); } elsif ($LogFormat eq '2') { # Same than "date time c-ip cs-username cs-method cs-uri-stem sc-status sc-bytes cs-version cs(User-Agent) cs(Referer)" $PerlParsingFormat="(\\S+ \\S+) (\\S+) (\\S+) (\\S+) (\\S+) ([\\d|-]+) ([\\d|-]+) \\S+ (\\S+) (\\S+)"; $pos_date=0;$pos_host=1;$pos_logname=2;$pos_method=3;$pos_url=4;$pos_code=5;$pos_size=6;$pos_agent=7;$pos_referer=8; @fieldlib=('date','host','logname','method','url','code','size','ua','referer'); } elsif ($LogFormat eq '3') { $PerlParsingFormat="([^\\t]*\\t[^\\t]*)\\t([^\\t]*)\\t([\\d|-]*)\\t([^\\t]*)\\t([^\\t]*)\\t([^\\t]*)\\t[^\\t]*\\t([^\\t]*)\\t([\\d]*)"; $pos_date=0;$pos_method=1;$pos_code=2;$pos_host=3;$pos_agent=4;$pos_referer=5;$pos_url=6;$pos_size=7; @fieldlib=('date','method','code','host','ua','referer','url','size'); } elsif ($LogFormat eq '4') { # Same than "%h %l %u %t \"%r\" %>s %b" # %u (user) is "(.+)" instead of "[^ ]+" because can contain space (Lotus Notes). $PerlParsingFormat="([^ ]+) [^ ]+ (.+) \\[([^ ]+) [^ ]+\\] \\\"([^ ]+) ([^ ]+) [^\\\"]+\\\" ([\\d|-]+) ([\\d|-]+)"; $pos_host=0;$pos_logname=1;$pos_date=2;$pos_method=3;$pos_url=4;$pos_code=5;$pos_size=6; @fieldlib=('host','logname','date','method','url','code','size'); } # This is a deprecated option, will be removed in a next version. elsif ($LogFormat eq '5') { # Same than "c-ip cs-username c-agent sc-authenticated date time s-svcname s-computername cs-referred r-host r-ip r-port time-taken cs-bytes sc-bytes cs-protocol cs-transport s-operation cs-uri cs-mime-type s-object-source sc-status s-cache-info" $PerlParsingFormat="([^\\t]*)\\t([^\\t]*)\\t([^\\t]*)\\t[^\\t]*\\t([^\\t]*\\t[^\\t]*)\\t[^\\t]*\\t[^\\t]*\\t([^\\t]*)\\t[^\\t]*\\t[^\\t]*\\t[^\\t]*\\t[^\\t]*\\t[^\\t]*\\t([^\\t]*)\\t[^\\t]*\\t[^\\t]*\\t([^\\t]*)\\t([^\\t]*)\\t[^\\t]*\\t[^\\t]*\\t([^\\t]*)\\t[^\\t]*"; $pos_host=0;$pos_logname=1;$pos_agent=2;$pos_date=3;$pos_referer=4;$pos_size=5;$pos_method=6;$pos_url=7;$pos_code=8; @fieldlib=('host','logname','ua','date','referer','size','method','url','code'); } } else { # Personalized log format my $LogFormatString=$LogFormat; # Replacement for Notes format string that are not Apache $LogFormatString =~ s/%vh/%virtualname/g; # Replacement for Apache format string $LogFormatString =~ s/%v(\s)/%virtualname$1/g; $LogFormatString =~ s/%v$/%virtualname/g; $LogFormatString =~ s/%h(\s)/%host$1/g; $LogFormatString =~ s/%h$/%host/g; $LogFormatString =~ s/%l(\s)/%other$1/g; $LogFormatString =~ s/%l$/%other/g; $LogFormatString =~ s/\"%u\"/%lognamequot/g; $LogFormatString =~ s/%u(\s)/%logname$1/g; $LogFormatString =~ s/%u$/%logname/g; $LogFormatString =~ s/%t(\s)/%time1$1/g; $LogFormatString =~ s/%t$/%time1/g; $LogFormatString =~ s/\"%r\"/%methodurl/g; $LogFormatString =~ s/%>s/%code/g; $LogFormatString =~ s/%b(\s)/%bytesd$1/g; $LogFormatString =~ s/%b$/%bytesd/g; $LogFormatString =~ s/\"%{Referer}i\"/%refererquot/g; $LogFormatString =~ s/\"%{User-Agent}i\"/%uaquot/g; $LogFormatString =~ s/%{mod_gzip_input_size}n/%gzipin/g; $LogFormatString =~ s/%{mod_gzip_output_size}n/%gzipout/g; $LogFormatString =~ s/%{mod_gzip_compression_ratio}n/%gzipratio/g; $LogFormatString =~ s/\(%{ratio}n\)/%deflateratio/g; # Replacement for a IIS and ISA format string $LogFormatString =~ s/cs-uri-query/%query/g; # Must be before cs-uri $LogFormatString =~ s/date\stime/%time2/g; $LogFormatString =~ s/c-ip/%host/g; $LogFormatString =~ s/cs-username/%logname/g; $LogFormatString =~ s/cs-method/%method/g; # GET, POST, SMTP, RETR STOR $LogFormatString =~ s/cs-uri-stem/%url/g; $LogFormatString =~ s/cs-uri/%url/g; $LogFormatString =~ s/sc-status/%code/g; $LogFormatString =~ s/sc-bytes/%bytesd/g; $LogFormatString =~ s/cs-version/%other/g; # Protocol $LogFormatString =~ s/cs\(User-Agent\)/%ua/g; $LogFormatString =~ s/c-agent/%ua/g; $LogFormatString =~ s/cs\(Referer\)/%referer/g; $LogFormatString =~ s/cs-referred/%referer/g; $LogFormatString =~ s/sc-authenticated/%other/g; $LogFormatString =~ s/s-svcname/%other/g; $LogFormatString =~ s/s-computername/%other/g; $LogFormatString =~ s/r-host/%virtualname/g; $LogFormatString =~ s/r-ip/%other/g; $LogFormatString =~ s/r-port/%other/g; $LogFormatString =~ s/time-taken/%other/g; $LogFormatString =~ s/cs-bytes/%other/g; $LogFormatString =~ s/cs-protocol/%other/g; $LogFormatString =~ s/cs-transport/%other/g; $LogFormatString =~ s/s-operation/%method/g; # GET, POST, SMTP, RETR STOR $LogFormatString =~ s/cs-mime-type/%other/g; $LogFormatString =~ s/s-object-source/%other/g; $LogFormatString =~ s/s-cache-info/%other/g; # Added for MMS $LogFormatString =~ s/protocol/%protocolmms/g; # cs-method might not be available $LogFormatString =~ s/c-status/%codemms/g; # c-status used when sc-status not available if ($Debug) { debug(" LogFormatString=$LogFormatString"); } # $LogFormatString has an AWStats format, so we can generate PerlParsingFormat variable my $i = 0; my $LogSeparatorWithoutStar=$LogSeparator; $LogSeparatorWithoutStar =~ s/[\*\+]//g; foreach my $f (split(/\s+/,$LogFormatString)) { # Add separator for next field if ($PerlParsingFormat) { $PerlParsingFormat.="$LogSeparator"; } # Special for logname if ($f =~ /%lognamequot$/) { $pos_logname = $i; $i++; push @fieldlib, 'logname'; $PerlParsingFormat .= "\\\"?([^\\\"]*)\\\"?"; # logname can be "value", "" and - in same log (Lotus notes) } # Date format elsif ($f =~ /%time1$/ || $f =~ /%time1b$/) { # [dd/mmm/yyyy:hh:mm:ss +0000] or [dd/mmm/yyyy:hh:mm:ss], time1b kept for backward compatibility $pos_date = $i; $i++; push @fieldlib, 'date'; $pos_tz = $i; $i++; push @fieldlib, 'tz'; $PerlParsingFormat .= "\\[([^$LogSeparatorWithoutStar]+)( [^$LogSeparatorWithoutStar]+)?\\]"; } elsif ($f =~ /%time2$/) { # yyyy-mm-dd hh:mm:ss $pos_date = $i; $i++; push @fieldlib, 'date'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+\\s[^$LogSeparatorWithoutStar]+)"; # Need \s for Exchange log files } elsif ($f =~ /%time3$/) { # mon d hh:mm:ss or mon dd hh:mm:ss yyyy or day mon dd hh:mm:ss or day mon dd hh:mm:ss yyyy $pos_date = $i; $i++; push @fieldlib, 'date'; $PerlParsingFormat .= "(?:\\w\\w\\w )?(\\w\\w\\w \\s?\\d+ \\d\\d:\\d\\d:\\d\\d(?: \\d\\d\\d\\d)?)"; } elsif ($f =~ /%time4$/) { # ddddddddddddd $pos_date = $i; $i++; push @fieldlib, 'date'; $PerlParsingFormat .= "(\\d+)"; } # Special for methodurl and methodurlnoprot elsif ($f =~ /%methodurl$/) { $pos_method = $i; $i++; push @fieldlib, 'method'; $pos_url = $i; $i++; push @fieldlib, 'url'; $PerlParsingFormat .= "\\\"([^$LogSeparatorWithoutStar]+) ([^$LogSeparatorWithoutStar]+) [^\\\"]+\\\""; } elsif ($f =~ /%methodurlnoprot$/) { $pos_method = $i; $i++; push @fieldlib, 'method'; $pos_url = $i; $i++; push @fieldlib, 'url'; $PerlParsingFormat .= "\\\"([^$LogSeparatorWithoutStar]+) ([^$LogSeparatorWithoutStar]+)\\\""; } # Common command tags elsif ($f =~ /%virtualname$/) { $pos_vh = $i; $i++; push @fieldlib, 'vhost'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%host_r$/) { $pos_hostr = $i; $i++; push @fieldlib, 'hostr'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%host$/) { $pos_host = $i; $i++; push @fieldlib, 'host'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%logname$/) { $pos_logname = $i; $i++; push @fieldlib, 'logname'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%method$/) { $pos_method = $i; $i++; push @fieldlib, 'method'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%url$/) { $pos_url = $i; $i++; push @fieldlib, 'url'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%query$/) { $pos_query = $i; $i++; push @fieldlib, 'query'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%code$/) { $pos_code = $i; $i++; push @fieldlib, 'code'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%bytesd$/) { $pos_size = $i; $i++; push @fieldlib, 'size'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%refererquot$/) { $pos_referer = $i; $i++; push @fieldlib, 'referer'; $PerlParsingFormat .= "\\\"([^\\\"]*)\\\""; # referer might be "" } elsif ($f =~ /%referer$/) { $pos_referer = $i; $i++; push @fieldlib, 'referer'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%uaquot$/) { $pos_agent = $i; $i++; push @fieldlib, 'ua'; $PerlParsingFormat .= "\\\"([^\\\"]*)\\\""; # ua might be "" } elsif ($f =~ /%uabracket$/) { $pos_agent = $i; $i++; push @fieldlib, 'ua'; $PerlParsingFormat .= "\\\[([^\\\]]*)\\\]"; # ua might be [] } elsif ($f =~ /%ua$/) { $pos_agent = $i; $i++; push @fieldlib, 'ua'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%gzipin$/ ) { $pos_gzipin=$i;$i++; push @fieldlib, 'gzipin'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%gzipout/ ) { # Compare $f to /%gzipout/ and not to /%gzipout$/ like other fields $pos_gzipout=$i;$i++; push @fieldlib, 'gzipout'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%gzipratio/ ) { # Compare $f to /%gzipratio/ and not to /%gzipratio$/ like other fields $pos_compratio=$i;$i++; push @fieldlib, 'gzipratio'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%deflateratio/ ) { # Compare $f to /%deflateratio/ and not to /%deflateratio$/ like other fields $pos_compratio=$i;$i++; push @fieldlib, 'deflateratio'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%email_r$/) { $pos_emailr = $i; $i++; push @fieldlib, 'email_r'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%email$/) { $pos_emails = $i; $i++; push @fieldlib, 'email'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%cluster$/) { $pos_cluster = $i; $i++; push @fieldlib, 'clusternb'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } elsif ($f =~ /%timetaken$/) { $pos_timetaken = $i; $i++; push @fieldlib, 'timetaken'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } # Special for protocolmms, used for method if method not already found (for MMS) elsif ($f =~ /%protocolmms$/) { if ($pos_method < 0) { $pos_method = $i; $i++; push @fieldlib, 'method'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } } # Special for codemms, used for code only if code not already found (for MMS) elsif ($f =~ /%codemms$/) { if ($pos_code < 0) { $pos_code = $i; $i++; push @fieldlib, 'code'; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } } # Extra tag elsif ($f =~ /%extra(\d+)$/) { $pos_extra[$1] = $i; $i++; push @fieldlib, "extra$1"; $PerlParsingFormat .= "([^$LogSeparatorWithoutStar]+)"; } # Other tag elsif ($f =~ /%other$/) { $PerlParsingFormat .= "[^$LogSeparatorWithoutStar]+"; } elsif ($f =~ /%otherquot$/) { $PerlParsingFormat .= "\\\"[^\\\"]*\\\""; } # Unknown tag (no parenthesis) else { $PerlParsingFormat .= "[^$LogSeparatorWithoutStar]+"; } } if (! $PerlParsingFormat) { error("No recognized format tag in personalized LogFormat string"); } } if ($pos_host < 0) { error("Your personalized LogFormat does not include all fields required by AWStats (Add \%host in your LogFormat string)."); } if ($pos_date < 0) { error("Your personalized LogFormat does not include all fields required by AWStats (Add \%time1 or \%time2 in your LogFormat string)."); } if ($pos_method < 0) { error("Your personalized LogFormat does not include all fields required by AWStats (Add \%methodurl or \%method in your LogFormat string)."); } if ($pos_url < 0) { error("Your personalized LogFormat does not include all fields required by AWStats (Add \%methodurl or \%url in your LogFormat string)."); } if ($pos_code < 0) { error("Your personalized LogFormat does not include all fields required by AWStats (Add \%code in your LogFormat string)."); } if ($pos_size < 0) { error("Your personalized LogFormat does not include all fields required by AWStats (Add \%bytesd in your LogFormat string)."); } $PerlParsingFormat=qr/^$PerlParsingFormat/; if ($Debug) { debug(" PerlParsingFormat is $PerlParsingFormat"); } } sub ShowMenuCateg { my ($categ,$categtext,$categicon,$frame,$targetpage,$linkanchor,$NewLinkParams,$NewLinkTarget)=(shift,shift,shift,shift,shift,shift,shift,shift); $categicon=''; # Comment this to enabme category icons my ($menu,$menulink,$menutext)=(shift,shift,shift); my $linetitle=0; # Call to plugins' function AddHTMLMenuLink foreach my $pluginname (keys %{$PluginsLoaded{'AddHTMLMenuLink'}}) { my $function="AddHTMLMenuLink_$pluginname('$categ',\$menu,\$menulink,\$menutext)"; eval("$function"); } foreach my $key (%$menu) { if ($menu->{$key}>0) { $linetitle++; last; } } if (! $linetitle) { return; } # At least one entry in menu for this category, we can show categpry and entries my $WIDTHMENU1=($FrameName eq 'mainleft'?$FRAMEWIDTH:150); print "".($categicon?" ":"")."$categtext:\n"; print ($frame?"\n":""); foreach my $key (sort { $menu->{$a} <=> $menu->{$b} } keys %$menu) { if ($menu->{$key}==0) { next; } if ($menulink->{$key}==1) { print ($frame?"":""); print "$menutext->{$key}"; print ($frame?"\n":"   "); } if ($menulink->{$key}==2) { print ($frame?"   \"...\" ":""); print "$menutext->{$key}\n"; print ($frame?"\n":"   "); } } print ($frame?"":"\n"); } sub ShowEmailSendersChart { my $NewLinkParams=shift; my $NewLinkTarget=shift; my $MaxLengthOfShownEMail=48; my $total_p;my $total_h;my $total_k; my $max_p;my $max_h;my $max_k; my $rest_p;my $rest_h;my $rest_k; # Show filter form #&ShowFormFilter("emailsfilter",$EmailsFilter); # Show emails list print "$Center 
\n"; my $title; if ($HTMLOutput{'allemails'} || $HTMLOutput{'lastemails'}) { $title="$Message[131]"; } else { $title="$Message[131] ($Message[77] $MaxNbOf{'EMailsShown'})   -   $Message[80]"; if ($ShowEMailSenders =~ /L/i) { $title.="   -   $Message[9]"; } } &tab_head("$title",19,0,'emailsenders'); print "$Message[131] : ".(scalar keys %_emails_h).""; if ($ShowEMailSenders =~ /H/i) { print "$Message[57]"; } if ($ShowEMailSenders =~ /B/i) { print "$Message[75]"; } if ($ShowEMailSenders =~ /M/i) { print "$Message[106]"; } if ($ShowEMailSenders =~ /L/i) { print "$Message[9]"; } print "\n"; print "Local External"; $total_p=$total_h=$total_k=0; $max_h=1; foreach (values %_emails_h) { if ($_ > $max_h) { $max_h = $_; } } $max_k=1; foreach (values %_emails_k) { if ($_ > $max_k) { $max_k = $_; } } my $count=0; if (! $HTMLOutput{'allemails'} && ! $HTMLOutput{'lastemails'}) { &BuildKeyList($MaxNbOf{'EMailsShown'},$MinHit{'EMail'},\%_emails_h,\%_emails_h); } if ($HTMLOutput{'allemails'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'EMail'},\%_emails_h,\%_emails_h); } if ($HTMLOutput{'lastemails'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'EMail'},\%_emails_h,\%_emails_l); } foreach my $key (@keylist) { my $newkey=$key; if (length($key)>$MaxLengthOfShownEMail) { $newkey=substr($key,0,$MaxLengthOfShownEMail)."..."; } my $bredde_h=0;my $bredde_k=0; if ($max_h > 0) { $bredde_h=int($BarWidth*$_emails_h{$key}/$max_h)+1; } if ($max_k > 0) { $bredde_k=int($BarWidth*$_emails_k{$key}/$max_k)+1; } print ""; my $direction=IsLocalEMail($key); if ($direction > 0) { print "$newkey-> "; } if ($direction == 0) { print "$newkey"; } if ($direction < 0) { print " <-$newkey"; } if ($ShowEMailSenders =~ /H/i) { print "$_emails_h{$key}"; } if ($ShowEMailSenders =~ /B/i) { print "".Format_Bytes($_emails_k{$key}).""; } if ($ShowEMailSenders =~ /M/i) { print "".Format_Bytes($_emails_k{$key}/($_emails_h{$key}||1)).""; } if ($ShowEMailSenders =~ /L/i) { print "".($_emails_l{$key}?Format_Date($_emails_l{$key},1):'-').""; } print "\n"; #$total_p += $_emails_p{$key}; $total_h += $_emails_h{$key}; $total_k += $_emails_k{$key}; $count++; } $rest_p=0; # $rest_p=$TotalPages-$total_p; $rest_h=$TotalHits-$total_h; $rest_k=$TotalBytes-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other sender emails print "$Message[2]"; if ($ShowEMailSenders =~ /H/i) { print "$rest_h"; } if ($ShowEMailSenders =~ /B/i) { print "".Format_Bytes($rest_k).""; } if ($ShowEMailSenders =~ /M/i) { print "".Format_Bytes($rest_k/($rest_h||1)).""; } if ($ShowEMailSenders =~ /L/i) { print " "; } print "\n"; } &tab_end(); } sub ShowEmailReceiversChart { my $NewLinkParams=shift; my $NewLinkTarget=shift; my $MaxLengthOfShownEMail=48; my $total_p;my $total_h;my $total_k; my $max_p;my $max_h;my $max_k; my $rest_p;my $rest_h;my $rest_k; # Show filter form #&ShowFormFilter("emailrfilter",$EmailrFilter); # Show emails list print "$Center 
\n"; my $title; if ($HTMLOutput{'allemailr'} || $HTMLOutput{'lastemailr'}) { $title="$Message[132]"; } else { $title="$Message[132] ($Message[77] $MaxNbOf{'EMailsShown'})   -   $Message[80]"; if ($ShowEMailReceivers =~ /L/i) { $title.="   -   $Message[9]"; } } &tab_head("$title",19,0,'emailreceivers'); print "$Message[132] : ".(scalar keys %_emailr_h).""; if ($ShowEMailReceivers =~ /H/i) { print "$Message[57]"; } if ($ShowEMailReceivers =~ /B/i) { print "$Message[75]"; } if ($ShowEMailReceivers =~ /M/i) { print "$Message[106]"; } if ($ShowEMailReceivers =~ /L/i) { print "$Message[9]"; } print "\n"; print "Local External"; $total_p=$total_h=$total_k=0; $max_h=1; foreach (values %_emailr_h) { if ($_ > $max_h) { $max_h = $_; } } $max_k=1; foreach (values %_emailr_k) { if ($_ > $max_k) { $max_k = $_; } } my $count=0; if (! $HTMLOutput{'allemailr'} && ! $HTMLOutput{'lastemailr'}) { &BuildKeyList($MaxNbOf{'EMailsShown'},$MinHit{'EMail'},\%_emailr_h,\%_emailr_h); } if ($HTMLOutput{'allemailr'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'EMail'},\%_emailr_h,\%_emailr_h); } if ($HTMLOutput{'lastemailr'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'EMail'},\%_emailr_h,\%_emailr_l); } foreach my $key (@keylist) { my $newkey=$key; if (length($key)>$MaxLengthOfShownEMail) { $newkey=substr($key,0,$MaxLengthOfShownEMail)."..."; } my $bredde_h=0;my $bredde_k=0; if ($max_h > 0) { $bredde_h=int($BarWidth*$_emailr_h{$key}/$max_h)+1; } if ($max_k > 0) { $bredde_k=int($BarWidth*$_emailr_k{$key}/$max_k)+1; } print ""; my $direction=IsLocalEMail($key); if ($direction > 0) { print "$newkey<- "; } if ($direction == 0) { print "$newkey"; } if ($direction < 0) { print " ->$newkey"; } if ($ShowEMailReceivers =~ /H/i) { print "$_emailr_h{$key}"; } if ($ShowEMailReceivers =~ /B/i) { print "".Format_Bytes($_emailr_k{$key}).""; } if ($ShowEMailReceivers =~ /M/i) { print "".Format_Bytes($_emailr_k{$key}/($_emailr_h{$key}||1)).""; } if ($ShowEMailReceivers =~ /L/i) { print "".($_emailr_l{$key}?Format_Date($_emailr_l{$key},1):'-').""; } print "\n"; #$total_p += $_emailr_p{$key}; $total_h += $_emailr_h{$key}; $total_k += $_emailr_k{$key}; $count++; } $rest_p=0; # $rest_p=$TotalPages-$total_p; $rest_h=$TotalHits-$total_h; $rest_k=$TotalBytes-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other receiver emails print "$Message[2]"; if ($ShowEMailReceivers =~ /H/i) { print "$rest_h"; } if ($ShowEMailReceivers =~ /B/i) { print "".Format_Bytes($rest_k).""; } if ($ShowEMailReceivers =~ /M/i) { print "".Format_Bytes($rest_k/($rest_h||1)).""; } if ($ShowEMailReceivers =~ /L/i) { print " "; } print "\n"; } &tab_end(); } #------------------------------------------------------------------------------ # Function: Calculate hash value of a string # Parameters: $str # Input: None # Output: None # Return: Hash value # Author: E-Lane #------------------------------------------------------------------------------ sub function_hash { my $str=shift(); my $c=substr $str,0 ,1; my $i=1; my $sum=0; while ($c ne '') { $sum+=ord($c); $c=substr $str,$i,1; $i++; } return $sum; } #------------------------------------------------------------------------------ # Function: Change all values %XX to its ASCII char # Parameters: $str # Input: None # Output: None # Return: String with ASCII chars # Author: E-Lane #------------------------------------------------------------------------------ sub ChangeToASCII { my $stringtochange=shift; $stringtochange =~ s/\%(..)/pack("c",hex($1))/ge; $stringtochange =~ s/\+/ /g; $stringtochange =~ s/\s+/ /g; return $stringtochange; } #------------------------------------------------------------------------------ # MAIN #------------------------------------------------------------------------------ ($DIR=$0) =~ s/([^\/\\]+)$//; ($PROG=$1) =~ s/\.([^\.]*)$//; $Extension=$1; $DIR||='.'; $DIR =~ s/([^\/\\])[\\\/]+$/$1/; $starttime=time(); # Get current time (time when AWStats was started) ($nowsec,$nowmin,$nowhour,$nowday,$nowmonth,$nowyear,$nowwday,$nowyday) = localtime($starttime); $nowweekofmonth=int($nowday/7); $nowweekofyear=int(($nowyday-1+6-($nowwday==0?6:$nowwday-1))/7)+1; if ($nowweekofyear > 52) { $nowweekofyear = 1; } $nowdaymod=$nowday%7; $nowwday++; $nowns=Time::Local::timegm(0,0,0,$nowday,$nowmonth,$nowyear); if ($nowdaymod <= $nowwday) { if (($nowwday != 7) || ($nowdaymod != 0)) { $nowweekofmonth=$nowweekofmonth+1; } } if ($nowdaymod > $nowwday) { $nowweekofmonth=$nowweekofmonth+2; } # Change format of time variables $nowweekofmonth="0$nowweekofmonth"; if ($nowweekofyear < 10) { $nowweekofyear = "0$nowweekofyear"; } if ($nowyear < 100) { $nowyear+=2000; } else { $nowyear+=1900; } $nowsmallyear=$nowyear;$nowsmallyear =~ s/^..//; if (++$nowmonth < 10) { $nowmonth = "0$nowmonth"; } if ($nowday < 10) { $nowday = "0$nowday"; } if ($nowhour < 10) { $nowhour = "0$nowhour"; } if ($nowmin < 10) { $nowmin = "0$nowmin"; } if ($nowsec < 10) { $nowsec = "0$nowsec"; } $nowtime=int($nowyear.$nowmonth.$nowday.$nowhour.$nowmin.$nowsec); # Get tomorrow time (will be used to discard some record with corrupted date (future date)) my ($tomorrowsec,$tomorrowmin,$tomorrowhour,$tomorrowday,$tomorrowmonth,$tomorrowyear) = localtime($starttime+86400); if ($tomorrowyear < 100) { $tomorrowyear+=2000; } else { $tomorrowyear+=1900; } if (++$tomorrowmonth < 10) { $tomorrowmonth = "0$tomorrowmonth"; } if ($tomorrowday < 10) { $tomorrowday = "0$tomorrowday"; } if ($tomorrowhour < 10) { $tomorrowhour = "0$tomorrowhour"; } if ($tomorrowmin < 10) { $tomorrowmin = "0$tomorrowmin"; } if ($tomorrowsec < 10) { $tomorrowsec = "0$tomorrowsec"; } $tomorrowtime=int($tomorrowyear.$tomorrowmonth.$tomorrowday.$tomorrowhour.$tomorrowmin.$tomorrowsec); # Allowed option #Esto no srive para nada, porque no lo vuelven a usar, pero quedan registradas las opciones posibles. my @AllowedCLIArgs=('migrate','config', 'logfile','output','runascli','update', 'staticlinks','staticlinksext','noloadplugin','loadplugin', 'hostfilter','urlfilter','refererpagesfilter', 'lang','month','year','framename','debug', 'showsteps','showdropped','showcorrupted','showunknownorigin', 'limitflush','confdir','updatefor', 'hostfilter','hostfilterex','urlfilter','urlfilterex','refererpagesfilter','refererpagesfilterex', 'pluginmode','filterrawlog','onlylines','onlyusers','onlyfiles','sitedomain'); #Creamos la lista para ordenar diferentes tipos my @listaOrd; $QueryString=''; # AWStats use GATEWAY_INTERFACE to known if ran as CLI or CGI. AWSTATS_DEL_GATEWAY_INTERFACE can # be set to force AWStats to be ran as CLI even from a web page. if ($ENV{'AWSTATS_DEL_GATEWAY_INTERFACE'}) { $ENV{'GATEWAY_INTERFACE'}=''; } if ($ENV{'GATEWAY_INTERFACE'}) { # Run from a browser as CGI # Prepare QueryString if ($ENV{'CONTENT_LENGTH'}) { binmode STDIN; read(STDIN, $QueryString, $ENV{'CONTENT_LENGTH'}); } if ($ENV{'QUERY_STRING'}) { $QueryString = $ENV{'QUERY_STRING'}; } $QueryString = CleanFromCSSA($QueryString); #Modified by E-Lane $QueryString = ChangeToASCII($QueryString); # No update but report by default when run from a browser $UpdateStats=($QueryString=~/update=1/i?1:0); if ($QueryString =~ /config=([^&]+)/i) { $SiteConfig=&DecodeEncodedString("$1"); } if ($QueryString =~ /logfile=([^&]+)/i) { $LogFile=&DecodeEncodedString("$1"); } if ($QueryString =~ /diricons=([^&]+)/i) { $DirIcons=&DecodeEncodedString("$1"); } if ($QueryString =~ /pluginmode=([^&]+)/i) { $PluginMode=&DecodeEncodedString("$1"); } if ($QueryString =~ /configdir=([^&]+)/i) { $DirConfig=&DecodeEncodedString("$1"); } # All filters if ($QueryString =~ /hostfilter=([^&]+)/i) { $FilterIn{'host'}=&DecodeEncodedString("$1"); } # Filter on host list can also be defined with hostfilter=filter if ($QueryString =~ /hostfilterex=([^&]+)/i) { $FilterEx{'host'}=&DecodeEncodedString("$1"); } # if ($QueryString =~ /urlfilter=([^&]+)/i) { $FilterIn{'url'}=&DecodeEncodedString("$1"); } # Filter on URL list can also be defined with urlfilter=filter if ($QueryString =~ /urlfilterex=([^&]+)/i) { $FilterEx{'url'}=&DecodeEncodedString("$1"); } # if ($QueryString =~ /refererpagesfilter=([^&]+)/i) { $FilterIn{'refererpages'}=&DecodeEncodedString("$1"); } # Filter on referer list can also be defined with refererpagesfilter=filter if ($QueryString =~ /refererpagesfilterex=([^&]+)/i) { $FilterEx{'refererpages'}=&DecodeEncodedString("$1"); } # # All output if ($QueryString =~ /output=allhosts:([^&]+)/i) { $FilterIn{'host'}=&DecodeEncodedString("$1"); } # Filter on host list can be defined with output=allhosts:filter to reduce number of lines read and showed if ($QueryString =~ /output=lasthosts:([^&]+)/i) { $FilterIn{'host'}=&DecodeEncodedString("$1"); } # Filter on host list can be defined with output=lasthosts:filter to reduce number of lines read and showed if ($QueryString =~ /output=urldetail:([^&]+)/i) { $FilterIn{'url'}=&DecodeEncodedString("$1"); } # Filter on URL list can be defined with output=urldetail:filter to reduce number of lines read and showed if ($QueryString =~ /output=refererpages:([^&]+)/i) { $FilterIn{'refererpages'}=&DecodeEncodedString("$1"); } # Filter on referer list can be defined with output=refererpages:filter to reduce number of lines read and showed #All new Paramas OnlyXXX and sitedomain defined by E-Lane if ($QueryString =~ /onlylines=([^&]+)/i) { @listaOrd=sort {$a<=>$b} (split(/\s+/,$1)); foreach my $elem (@listaOrd) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="ol".function_hash($elem); } else { $FileSuffix.="ol$elem"; $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyLines, qr/$elem/i; } } } if ($QueryString =~ /onlycoms=([^&]+)/i) { @listaOrd=sort {$a<=>$b} (split(/\s+/,$1)); foreach my $elem (@listaOrd) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="ol".function_hash($elem); } else { $FileSuffix.="oc$elem"; $elem='.*community_id='.quotemeta($elem).'.*'; } if ($elem) { push @OnlyLines, qr/$elem/i; } } } if ($QueryString =~ /onlyusers=([^&]+)/i) { @listaOrd=sort {$a<=>$b} (split(/\s+/,$1)); foreach my $elem (@listaOrd) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="ou".function_hash($elem); } else { $FileSuffix.="ou$elem"; $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyUsers, qr/$elem/i; } } } if ($QueryString =~ /onlyfiles=([^&]+)/i) { @listaOrd=sort {$a<=>$b} (split(/\s+/,$1)); foreach my $elem (@listaOrd) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="of".function_hash($elem); } else { $FileSuffix.="of$elem"; $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyFiles, qr/$elem/i; } } # next; } if ($QueryString =~ /sitedomain=([^&]+)/i) { $SiteDomain=&DecodeEncodedString("$1"); } # If migrate if ($QueryString =~ /(^|-|&)migrate=([^&]+)/i) { $MigrateStats=&DecodeEncodedString("$2"); $MigrateStats =~ /^(.*)$PROG(\d{0,2})(\d\d)(\d\d\d\d)(.*)\.txt$/; $SiteConfig=$5?$5:'xxx'; $SiteConfig =~ s/^\.//; # SiteConfig is used to find config file } } else { # Run from command line # Prepare QueryString for (0..@ARGV-1) { # If migrate if ($ARGV[$_] =~ /(^|-|&)migrate=([^&]+)/i) { $MigrateStats="$2"; $MigrateStats =~ /^(.*)$PROG(\d{0,2})(\d\d)(\d\d\d\d)(.*)\.txt$/; $SiteConfig=$5?$5:'xxx'; $SiteConfig =~ s/^\.//; # SiteConfig is used to find config file next; } # TODO Check if ARGV is in @AllowedArg if ($QueryString) { $QueryString .= '&'; } my $NewLinkParams=$ARGV[$_]; $NewLinkParams =~ s/^-+//; $QueryString .= "$NewLinkParams"; } $QueryString = CleanFromCSSA($QueryString); #Modified by E-Lane $QueryString = ChangeToASCII($QueryString); # Update with no report by default when run from command line $UpdateStats=1; if ($QueryString =~ /config=([^&]+)/i) { $SiteConfig="$1"; } if ($QueryString =~ /logfile=([^&]+)/i) { $LogFile="$1"; } if ($QueryString =~ /diricons=([^&]+)/i) { $DirIcons="$1"; } if ($QueryString =~ /pluginmode=([^&]+)/i) { $PluginMode="$1"; } if ($QueryString =~ /configdir=([^&]+)/i) { $DirConfig="$1"; } # All filters if ($QueryString =~ /hostfilter=([^&]+)/i) { $FilterIn{'host'}="$1"; } # Filter on host list can also be defined with hostfilter=filter if ($QueryString =~ /hostfilterex=([^&]+)/i) { $FilterEx{'host'}="$1"; } # if ($QueryString =~ /urlfilter=([^&]+)/i) { $FilterIn{'url'}="$1"; } # Filter on URL list can also be defined with urlfilter=filter if ($QueryString =~ /urlfilterex=([^&]+)/i) { $FilterEx{'url'}="$1"; } # if ($QueryString =~ /refererpagesfilter=([^&]+)/i) { $FilterIn{'refererpages'}="$1"; } # Filter on referer list can also be defined with refererpagesfilter=filter if ($QueryString =~ /refererpagesfilterex=([^&]+)/i) { $FilterEx{'refererpages'}="$1"; } # # All output if ($QueryString =~ /output=allhosts:([^&]+)/i) { $FilterIn{'host'}="$1"; } # Filter on host list can be defined with output=allhosts:filter to reduce number of lines read and showed if ($QueryString =~ /output=lasthosts:([^&]+)/i) { $FilterIn{'host'}="$1"; } # Filter on host list can be defined with output=lasthosts:filter to reduce number of lines read and showed if ($QueryString =~ /output=urldetail:([^&]+)/i) { $FilterIn{'url'}="$1"; } # Filter on URL list can be defined with output=urldetail:filter to reduce number of lines read and showed if ($QueryString =~ /output=refererpages:([^&]+)/i) { $FilterIn{'refererpages'}="$1"; } # Filter on referer list can be defined with output=refererpages:filter to reduce number of lines read and showed #All new paramas OnlyXXX and sitedomain defined by E-Lane if ($QueryString =~ /onlylines=([^&]+)/i) { @listaOrd=sort {$a<=>$b} (split(/\s+/,$1)); foreach my $elem (@listaOrd) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="ol".function_hash($elem); } else { $FileSuffix.="ol$elem"; $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyLines, qr/$elem/i; } } # foreach my $elem (split(/\s+/,$1)) { # if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="ol".function_hash($elem); } # else { $FileSuffix.="ol$elem"; $elem='^'.quotemeta($elem).'$'; } # if ($elem) { push @OnlyLines, qr/$elem/i; } # } # next; } if ($QueryString =~ /onlyusers=([^&]+)/i) { @listaOrd=sort {$a<=>$b} (split(/\s+/,$1)); foreach my $elem (@listaOrd) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="ou".function_hash($elem); } else { $FileSuffix.="ou$elem"; $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyUsers, qr/$elem/i; } } # foreach my $elem (split(/\s+/,$1)) { # if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="ou".function_hash($elem); } # else { $FileSuffix.="ou$elem"; $elem='^'.quotemeta($elem).'$'; $FileSuffix.="$elem"; } # if ($elem) { push @OnlyUsers, qr/$elem/i; } # } # next; } if ($QueryString =~ /onlycoms=([^&]+)/i) { @listaOrd=sort {$a<=>$b} (split(/\s+/,$1)); foreach my $elem (@listaOrd) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="ol".function_hash($elem); } else { $FileSuffix.="oc$elem"; $elem='.*community_id='.quotemeta($elem).'.*'; } if ($elem) { push @OnlyLines, qr/$elem/i; } } } if ($QueryString =~ /onlyfiles=([^&]+)/i) { @listaOrd=sort {$a<=>$b} (split(/\s+/,$1)); foreach my $elem (@listaOrd) { if ($elem =~ /^REGEX\[(.*)\]$/i) { $elem=$1; $FileSuffix.="of".function_hash($elem); } else { $FileSuffix.="of$elem"; $elem='^'.quotemeta($elem).'$'; } if ($elem) { push @OnlyFiles, qr/$elem/i; } } # next; } if ($QueryString =~ /sitedomain=([^&]+)/i) { $SiteDomain=$1; } # If show options if ($QueryString =~ /showsteps/i) { $ShowSteps=1; $QueryString=~s/showsteps[^&]*//i; } if ($QueryString =~ /showcorrupted/i) { $ShowCorrupted=1; $QueryString=~s/showcorrupted[^&]*//i; } if ($QueryString =~ /showdropped/i) { $ShowDropped=1; $QueryString=~s/showdropped[^&]*//i; } if ($QueryString =~ /showunknownorigin/i) { $ShowUnknownOrigin=1; $QueryString=~s/showunknownorigin[^&]*//i; } } if ($QueryString =~ /(^|&)staticlinks/i) { $StaticLinks=".$SiteConfig"; } if ($QueryString =~ /(^|&)staticlinks=([^&]+)/i) { $StaticLinks=".$2"; } # When ran from awstatsbuildstaticpages.pl if ($QueryString =~ /(^|&)staticlinksext=([^&]+)/i) { $StaticExt="$2"; } if ($QueryString =~ /(^|&)framename=([^&]+)/i) { $FrameName="$2"; } if ($QueryString =~ /(^|&)debug=(\d+)/i) { $Debug=$2; } if ($QueryString =~ /(^|&)updatefor=(\d+)/i) { $UpdateFor=$2; } if ($QueryString =~ /(^|&)noloadplugin=([^&]+)/i) { foreach (split(/,/,$2)) { $NoLoadPlugin{"$_"}=1; } } if ($QueryString =~ /(^|&)loadplugin=([^&]+)/i) { foreach (split(/,/,$2)) { $NoLoadPlugin{"$_"}=-1; } } if ($QueryString =~ /(^|&)limitflush=(\d+)/i) { $LIMITFLUSH=$2; } # Get/Define output if ($QueryString =~ /(^|&)output(=[^&]*|)(.*)&output(=[^&]*|)(&|$)/i) { error("Only 1 output option is allowed","","",1); } if ($QueryString =~ /(^|&)output(=[^&]*|)(&|$)/i) { # At least one output expected. We define %HTMLOutput my $outputlist="$2"; if ($outputlist) { $outputlist =~ s/^=//; foreach my $outputparam (split(/,/,$outputlist)) { $outputparam=~s/:(.*)$//; if ($outputparam) { $HTMLOutput{lc($outputparam)}="$1"||1; } } } # If on command line and no update if (! $ENV{'GATEWAY_INTERFACE'} && $QueryString !~ /update/i) { $UpdateStats=0; } # If no output defined, used default value if (! scalar keys %HTMLOutput) { $HTMLOutput{'main'}=1; } } if ($ENV{'GATEWAY_INTERFACE'} && ! scalar keys %HTMLOutput) { $HTMLOutput{'main'}=1; } # Remove -output option with no = from QueryString $QueryString=~s/(^|&)output(&|$)/$1/i; $QueryString=~s/&+$//; # Check year and month parameters if ($QueryString =~ /(^|&)month=(year)/i) { error("month=year is a deprecated option. Use month=all instead."); } if ($QueryString =~ /(^|&)year=(\d\d\d\d)/i) { $YearRequired=sprintf("%04d",$2); } else { $YearRequired="$nowyear"; } if ($QueryString =~ /(^|&)month=(\d{1,2})/i) { $MonthRequired=sprintf("%02d",$2); } elsif ($QueryString =~ /(^|&)month=(all)/i) { $MonthRequired='all'; } else { $MonthRequired="$nowmonth"; } if ($QueryString =~ /(^|&)day=(\d{1,2})/i) { $DayRequired=sprintf("%02d",$2); } # day is a hidden option. Must not be used (Make results not understandable). Available for users that rename history files with day. else { $DayRequired=''; } # Check parameter validity # TODO # Print AWStats and Perl version if ($Debug) { debug(ucfirst($PROG)." - $VERSION - Perl $^X $]",1); debug("DIR=$DIR PROG=$PROG",2); debug("QUERY_STRING=$QueryString",2); debug("HTMLOutput=".join(',',keys %HTMLOutput),1); debug("YearRequired=$YearRequired, MonthRequired=$MonthRequired",2); debug("UpdateFor=$UpdateFor",2); debug("PluginMode=$PluginMode",2); debug("DirConfig=$DirConfig",2); } # Force SiteConfig if AWSTATS_FORCE_CONFIG is defined if ($ENV{'AWSTATS_CONFIG'}) { $ENV{'AWSTATS_FORCE_CONFIG'}=$ENV{'AWSTATS_CONFIG'}; } # For backward compatibility if ($ENV{'AWSTATS_FORCE_CONFIG'}) { if ($Debug) { debug("AWSTATS_FORCE_CONFIG parameter is defined to '".$ENV{'AWSTATS_FORCE_CONFIG'}."'. $PROG will use this as config value."); } $SiteConfig=$ENV{'AWSTATS_FORCE_CONFIG'}; } if ((! $ENV{'GATEWAY_INTERFACE'}) && (! $SiteConfig)) { &Read_Ref_Data('browsers','domains','operating_systems','robots','search_engines','worms'); print "----- $PROG $VERSION (c) 2000-2004 Laurent Destailleur -----\n"; print "AWStats is a free web server logfile analyzer to show you advanced web\n"; print "statistics.\n"; print "AWStats comes with ABSOLUTELY NO WARRANTY. It's a free software distributed\n"; print "with a GNU General Public License (See LICENSE file for details).\n"; print "\n"; print "Syntax: $PROG.$Extension -config=virtualhostname [options]\n"; print "\n"; print " This runs $PROG in command line to update statistics of a web site, from\n"; print " the log file defined in AWStats config file (with -update option), or build\n"; print " a HTML report (with -output option).\n"; print " First, $PROG tries to read $PROG.virtualhostname.conf as the config file.\n"; print " If not found, $PROG tries to read $PROG.conf\n"; print " Note 1: Config files ($PROG.virtualhostname.conf or $PROG.conf) must be\n"; print " in /etc/awstats, /usr/local/etc/awstats, /etc or same directory than\n"; print " awstats.pl script file.\n"; print " Note 2: If AWSTATS_FORCE_CONFIG environment variable is defined, AWStats will\n"; print " use it as the \"config\" value, whatever is the value on command line or URL.\n"; print " See AWStats documentation for all setup instrutions.\n"; print "\n"; print "Options to update statistics:\n"; print " -update to update statistics (default)\n"; print " -showsteps to add benchmark information every $NBOFLINESFORBENCHMARK lines processed\n"; print " -showcorrupted to add output for each corrupted lines found, with reason\n"; print " -showdropped to add output for each dropped lines found, with reason\n"; print " -logfile=x to change log to analyze whatever is 'LogFile' in config file\n"; print " -updatefor=n to stop the update process after parsing n lines\n"; print " Be care to process log files in chronological order when updating statistics.\n"; print "\n"; print "Options to show statistics:\n"; print " -output to output main HTML report (no update made except with -update)\n"; print " -output=x to output other report pages where x is:\n"; print " alldomains to build page of all domains/countries\n"; print " allhosts to build page of all hosts\n"; print " lasthosts to build page of last hits for hosts\n"; print " unknownip to build page of all unresolved IP\n"; print " allemails to build page of all email senders (maillog)\n"; print " lastemails to build page of last email senders (maillog)\n"; print " allemailr to build page of all email receivers (maillog)\n"; print " lastemailr to build page of last email receivers (maillog)\n"; print " alllogins to build page of all logins used\n"; print " lastlogins to build page of last hits for logins\n"; print " allrobots to build page of all robots/spider visits\n"; print " lastrobots to build page of last hits for robots\n"; print " urldetail to list most often viewed pages \n"; print " urldetail:filter to list most often viewed pages matching filter\n"; print " urlentry to list entry pages\n"; print " urlentry:filter to list entry pages matching filter\n"; print " urlexit to list exit pages\n"; print " urlexit:filter to list exit pages matching filter\n"; print " osdetail to build page with os detailed versions\n"; print " browserdetail to build page with browsers detailed versions\n"; print " unknownbrowser to list 'User Agents' with unknown browser\n"; print " unknownos to list 'User Agents' with unknown OS\n"; print " refererse to build page of all refering search engines\n"; print " refererpages to build page of all refering pages\n"; #print " referersites to build page of all refering sites\n"; print " keyphrases to list all keyphrases used on search engines\n"; print " keywords to list all keywords used on search engines\n"; print " errors404 to list 'Referers' for 404 errors\n"; print " -staticlinks to have static links in HTML report page\n"; print " -staticlinksext=xxx to have static links with .xxx extension instead of .html\n"; print " -lang=LL to output a HTML report in language LL (en,de,es,fr,it,nl,...)\n"; print " -month=MM to output a HTML report for an old month MM\n"; print " -year=YYYY to output a HTML report for an old year YYYY\n"; print " Those 'date' options doesn't allow you to process old log file. They only\n"; print " allow you to see a past report for a chosen month/year period instead of\n"; print " current month/year.\n"; print "\n"; print "Other options:\n"; print " -debug=X to add debug informations lesser than level X (speed reduced)\n"; print "\n"; print "Now supports/detects:\n"; print " Web/Ftp/Mail log analyze (and load balanced log files)\n"; print " Reverse DNS lookup (IPv4 and IPv6) and GeoIP lookup\n"; print " Number of visits, number of unique visitors\n"; print " Visits duration and list of last visits\n"; print " Authenticated users\n"; print " Days of week and rush hours\n"; print " Hosts list and unresolved IP addresses list\n"; print " Most viewed, entry and exit pages\n"; print " Files type and Web compression (mod_gzip, mod_deflate stats)\n"; print " Screen size\n"; print " Number of times site is 'added to favorites bookmarks'\n"; print " Ratio of Browsers with support of: Java, Flash, RealG2 reader,\n"; print " Quicktime reader, WMA reader, PDF reader\n"; print " Configurable personalized reports\n"; print " ".(scalar keys %DomainsHashIDLib)." domains/countries\n"; print " ".(scalar keys %RobotsHashIDLib)." robots\n"; print " ".(scalar keys %WormsHashLib)." worm's families\n"; print " ".(scalar keys %OSHashLib)." operating systems\n"; print " ".(scalar keys %BrowsersHashIDLib)." browsers\n"; print " ".(scalar keys %SearchEnginesHashLib)." search engines (and keyphrases/keywords used from them)\n"; print " All HTTP errors with last referrer\n"; print " Report by day/month/year\n"; print " Dynamic or static HTML or XHTML reports, static PDF reports\n"; print " Indexed text or XML monthly database\n"; print " And a lot of other advanced features and options...\n"; print "New versions and FAQ at http://awstats.sourceforge.net\n"; exit 2; } $SiteConfig||=$ENV{'SERVER_NAME'}; #$ENV{'SERVER_NAME'}||=$SiteConfig; # For thoose who use __SERVER_NAME__ in conf file and use CLI. $ENV{'AWSTATS_CURRENT_CONFIG'}=$SiteConfig; # Read config file (SiteConfig must be defined) &Read_Config($DirConfig); # Check language if ($QueryString =~ /(^|&)lang=([^&]+)/i) { $Lang="$2"; } if (! $Lang || $Lang eq 'auto') { # If lang not defined or forced to auto my $langlist=$ENV{'HTTP_ACCEPT_LANGUAGE'}||''; $langlist =~ s/;[^,]*//g; if ($Debug) { debug("Search an available language among HTTP_ACCEPT_LANGUAGE=$langlist",1); } foreach my $code (split(/,/,$langlist)) { # Search for a valid lang in priority if ($LangBrowserToLangAwstats{$code}) { $Lang=$LangBrowserToLangAwstats{$code}; if ($Debug) { debug(" Will try to use Lang=$Lang",1); } last; } $code =~ s/-.*$//; if ($LangBrowserToLangAwstats{$code}) { $Lang=$LangBrowserToLangAwstats{$code}; if ($Debug) { debug(" Will try to use Lang=$Lang",1); } last; } } } if (! $Lang || $Lang eq 'auto') { if ($Debug) { debug(" No language defined or available. Will use Lang=en",1); } $Lang='en'; } # Check and correct bad parameters &Check_Config(); # Now SiteDomain is defined # Define frame name and correct variable for frames if (! $FrameName) { if ($ENV{'GATEWAY_INTERFACE'} && $UseFramesWhenCGI && $HTMLOutput{'main'} && ! $PluginMode) { $FrameName='index'; } else { $FrameName='main'; } } # Load Message files, Reference data files and Plugins if ($Debug) { debug("FrameName=$FrameName",1); } if ($FrameName ne 'index') { &Read_Language_Data($Lang); if ($FrameName ne 'mainleft') { my %datatoload=(); if ($UpdateStats) { # If update if ($LevelForFileTypesDetection<2) { $datatoload{'mime'}=1; } # Only if need to filter on known extensions if ($LevelForRobotsDetection) { $datatoload{'robots'}=1; } # ua if ($LevelForWormsDetection) { $datatoload{'worms'}=1; } # url if ($LevelForBrowsersDetection) { $datatoload{'browsers'}=1; } # ua if ($LevelForOSDetection) { $datatoload{'operating_systems'}=1; } # ua if ($LevelForRefererAnalyze) { $datatoload{'search_engines'}=1; } # referer # if (...) { $datatoload{'referer_spam'}=1; } } if (scalar keys %HTMLOutput) { # If output if ($ShowDomainsStats) { $datatoload{'domains'}=1; } if ($ShowFileTypesStats) { $datatoload{'mime'}=1; } if ($ShowRobotsStats) { $datatoload{'robots'}=1; } if ($ShowWormsStats) { $datatoload{'worms'}=1; } if ($ShowBrowsersStats) { $datatoload{'browsers'}=1; } if ($ShowOSStats) { $datatoload{'operating_systems'}=1; } if ($ShowOriginStats) { $datatoload{'search_engines'}=1; } if ($ShowHTTPErrorsStats) { $datatoload{'status_http'}=1; } if ($ShowSMTPErrorsStats) { $datatoload{'status_smtp'}=1; } } &Read_Ref_Data(keys %datatoload); } &Read_Plugins(); } # Here charset is defined, so we can send the http header (Need BuildReportFormat,PageCode) if (! $HeaderHTTPSent && $ENV{'GATEWAY_INTERFACE'}) { http_head(); } # Run from a browser as CGI # Init other parameters $NBOFLINESFORBENCHMARK--; if ($ENV{'GATEWAY_INTERFACE'}) { $DirCgi=''; } if ($DirCgi && !($DirCgi =~ /\/$/) && !($DirCgi =~ /\\$/)) { $DirCgi .= '/'; } if (! $DirData || $DirData =~ /^\./) { if (! $DirData || $DirData eq '.') { $DirData="$DIR"; } # If not defined or chosen to '.' value then DirData is current dir elsif ($DIR && $DIR ne '.') { $DirData="$DIR/$DirData"; } } $DirData||='.'; # If current dir not defined then we put it to '.' $DirData =~ s/[\\\/]+$//; if ($FirstDayOfWeek == 1) { @DOWIndex = (1,2,3,4,5,6,0); } else { @DOWIndex = (0,1,2,3,4,5,6); } # Should we link to ourselves or to a wrapper script $AWScript=($WrapperScript?"$WrapperScript":"$DirCgi$PROG.$Extension"); # Print html header (Need HTMLOutput,Expires,Lang,StyleSheet,HTMLHeadSectionExpires defined by Read_Config, PageCode defined by Read_Language_Data) if (! $HeaderHTMLSent) { &html_head; } # AWStats output is replaced by a plugin output if ($PluginMode) { my $function="BuildFullHTMLOutput_$PluginMode()"; eval("$function"); if ($? || $@) { error("$@"); } &html_end(0); exit 0; } # Security check if ($AllowAccessFromWebToAuthenticatedUsersOnly && $ENV{'GATEWAY_INTERFACE'}) { if ($Debug) { debug("REMOTE_USER=".$ENV{"REMOTE_USER"}); } if (! $ENV{"REMOTE_USER"}) { error("Access to statistics is only allowed from an authenticated session to authenticated users."); } if (@AllowAccessFromWebToFollowingAuthenticatedUsers) { my $userisinlist=0; my $currentuser=qr/^$ENV{"REMOTE_USER"}$/i; $currentuser =~ s/\s/%20/g; # Allow authenticated user with space in name to be compared to allowed user list foreach (@AllowAccessFromWebToFollowingAuthenticatedUsers) { if (/$currentuser/o) { $userisinlist=1; last; } } if (! $userisinlist) { error("User '".$ENV{"REMOTE_USER"}."' is not allowed to access statistics of this domain/config."); } } } if ($AllowAccessFromWebToFollowingIPAddresses && $ENV{'GATEWAY_INTERFACE'}) { my $useripaddress=&Convert_IP_To_Decimal($ENV{"REMOTE_ADDR"}); my @allowaccessfromipaddresses = split (/[\s,]+/, $AllowAccessFromWebToFollowingIPAddresses); my $allowaccess = 0; foreach my $ipaddressrange (@allowaccessfromipaddresses) { if ($ipaddressrange !~ /^(\d+\.\d+\.\d+\.\d+)(?:-(\d+\.\d+\.\d+\.\d+))*$/) { error("AllowAccessFromWebToFollowingIPAddresses is defined to '$AllowAccessFromWebToFollowingIPAddresses' but does not match the correct syntax: IPAddressMin[-IPAddressMax]"); } my $ipmin=&Convert_IP_To_Decimal($1); my $ipmax=$2?&Convert_IP_To_Decimal($2):$ipmin; # Is it an authorized ip ? if (($useripaddress >= $ipmin) && ($useripaddress <= $ipmax)) { $allowaccess = 1; last; } } if (! $allowaccess) { error("Access to statistics is not allowed from your IP Address ".$ENV{"REMOTE_ADDR"}); } } if (($UpdateStats || $MigrateStats) && (! $AllowToUpdateStatsFromBrowser) && $ENV{'GATEWAY_INTERFACE'}) { error("".($UpdateStats?"Update":"Migrate")." of statistics has not been allowed from a browser (AllowToUpdateStatsFromBrowser should be set to 1)."); } if (scalar keys %HTMLOutput && $MonthRequired eq 'all') { if (! $AllowFullYearView) { error("Full year view has not been allowed (AllowFullYearView is set to 0)."); } if ($AllowFullYearView < 3 && $ENV{'GATEWAY_INTERFACE'}) { error("Full year view has not been allowed from a browser (AllowFullYearView should be set to 3)."); } } #------------------------------------------ # MIGRATE PROCESS (Must be after reading config cause we need MaxNbOf... and Min...) #------------------------------------------ if ($MigrateStats) { if ($Debug) { debug("MigrateStats is $MigrateStats",2); } if ($MigrateStats !~ /^(.*)$PROG(\d{0,2})(\d\d)(\d\d\d\d)(.*)\.txt$/) { error("AWStats history file name must match following syntax: ${PROG}MMYYYY[.config].txt","","",1); } $DirData="$1"; $DayRequired="$2"; $MonthRequired="$3"; $YearRequired="$4"; $FileSuffix="$5"; # Correct DirData if (! $DirData || $DirData =~ /^\./) { if (! $DirData || $DirData eq '.') { $DirData="$DIR"; } # If not defined or chosen to '.' value then DirData is current dir elsif ($DIR && $DIR ne '.') { $DirData="$DIR/$DirData"; } } $DirData||='.'; # If current dir not defined then we put it to '.' $DirData =~ s/[\\\/]+$//; print "Start migration for file '$MigrateStats'."; print $ENV{'GATEWAY_INTERFACE'}?"
\n":"\n"; if ($EnableLockForUpdate) { &Lock_Update(1); } my $newhistory=&Read_History_With_TmpUpdate($YearRequired,$MonthRequired,1,0,'all'); if (rename("$newhistory","$MigrateStats")==0) { unlink "$newhistory"; error("Failed to rename \"$newhistory\" into \"$MigrateStats\".\nWrite permissions on \"$MigrateStats\" might be wrong".($ENV{'GATEWAY_INTERFACE'}?" for a 'migration from web'":"")." or file might be opened."); } if ($EnableLockForUpdate) { &Lock_Update(0); } print "Migration for file '$MigrateStats' successful."; print $ENV{'GATEWAY_INTERFACE'}?"
\n":"\n"; &html_end(1); exit 0; } # Output main frame page and exit. This must be after the security check. if ($FrameName eq 'index') { # Define the NewLinkParams for main chart my $NewLinkParams=${QueryString}; $NewLinkParams =~ s/(^|&)framename=[^&]*//i; $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; if ($NewLinkParams) { $NewLinkParams="${NewLinkParams}&"; } # Exit if main frame print "\n"; print "\n"; print "\n"; print "<body>"; print "Your browser does not support frames.<br />\n"; print "You must set AWStats UseFramesWhenCGI parameter to 0\n"; print "to see your reports.<br />\n"; print "</body>\n"; print "\n"; &html_end(0); exit 0; } %MonthNumLib = ("01","$Message[60]","02","$Message[61]","03","$Message[62]","04","$Message[63]","05","$Message[64]","06","$Message[65]","07","$Message[66]","08","$Message[67]","09","$Message[68]","10","$Message[69]","11","$Message[70]","12","$Message[71]"); # Build ListOfYears list with all existing years if ($Debug) { debug("Scan for last history files into DirData='$DirData'"); } $lastyearbeforeupdate=0; #opendir(DIR,"$DirData"); #foreach (grep /^$PROG(\d\d)(\d\d\d\d)$FileSuffix\.txt(|\.gz)$/, sort readdir DIR) { # /^$PROG(\d\d)(\d\d\d\d)$FileSuffix\.txt(|\.gz)$/; # if (! $ListOfYears{"$2"} || "$1" gt $ListOfYears{"$2"}) { # $ListOfYears{"$2"}="$1"; # ListOfYears contains max month found # if ("$2" gt $lastyearbeforeupdate) { $lastyearbeforeupdate="$2"; } # } #} #close DIR; #commented by e-lane # Get value for LastLine if ($lastyearbeforeupdate) { # Read 'general' section of last history file for LastLine # &Read_History_With_TmpUpdate($lastyearbeforeupdate,$ListOfYears{$lastyearbeforeupdate},0,0,"general"); &Read_History_With_TmpUpdate($YearRequired,$MonthRequired,0,0,"general"); } if ($Debug) { debug("Last year=$lastyearbeforeupdate - Last month=$ListOfYears{$lastyearbeforeupdate}"); debug("LastLine=$LastLine"); debug("LastLineNumber=$LastLineNumber"); debug("LastLineOffset=$LastLineOffset"); debug("LastLineChecksum=$LastLineChecksum"); } # Init vars &Init_HashArray(); #------------------------------------------ # UPDATE PROCESS #------------------------------------------ my $lastlinenb=0; my $lastlineoffset=0; my $lastlineoffsetnext=0; if ($Debug) { debug("UpdateStats is $UpdateStats",2); } if ($UpdateStats && $FrameName ne 'index' && $FrameName ne 'mainleft') { # Update only on index page or when not framed to avoid update twice my %MonthNum = ("Jan","01","ene","01","Feb","02","feb","02","Mar","03","mar","03","Apr","04","abr","04","May","05","may","05","Jun","06","jun","06","Jul","07","jul","07","Aug","08","ago","08","Sep","09","sep","09","Oct","10","oct","10","Nov","11","nov","11","Dec","12","dic","12"); # MonthNum must be in english because used to translate log date in apache log files if (! scalar keys %HTMLOutput) { print "Update for config \"$FileConfig\"\n"; print "With data in log file \"$LogFile\"...\n"; } my $lastprocessedyear=$lastyearbeforeupdate; my $lastprocessedmonth=$ListOfYears{$lastyearbeforeupdate}||0; my $lastprocessedyearmonth=sprintf("%04i%02i",$lastprocessedyear,$lastprocessedmonth); my @list; # Init RobotsSearchIDOrder required for update process @list=(); if ($LevelForRobotsDetection >= 1) { foreach (1..$LevelForRobotsDetection) { push @list,"list$_"; } push @list,"listgen"; # Always added } foreach my $key (@list) { push @RobotsSearchIDOrder,@{"RobotsSearchIDOrder_$key"}; if ($Debug) { debug("Add ".@{"RobotsSearchIDOrder_$key"}." elements from RobotsSearchIDOrder_$key into RobotsSearchIDOrder",2); } } if ($Debug) { debug("RobotsSearchIDOrder has now ".@RobotsSearchIDOrder." elements",1); } # Init SearchEnginesIDOrder required for update process @list=(); if ($LevelForSearchEnginesDetection >= 1) { foreach (1..$LevelForSearchEnginesDetection) { push @list,"list$_"; } push @list,"listgen"; # Always added } foreach my $key (@list) { push @SearchEnginesSearchIDOrder,@{"SearchEnginesSearchIDOrder_$key"}; if ($Debug) { debug("Add ".@{"SearchEnginesSearchIDOrder_$key"}." elements from SearchEnginesSearchIDOrder_$key into SearchEnginesSearchIDOrder",2); } } if ($Debug) { debug("SearchEnginesSearchIDOrder has now ".@SearchEnginesSearchIDOrder." elements",1); } # Complete HostAliases array my $sitetoanalyze=quotemeta(lc($SiteDomain)); if (! @HostAliases) { warning("Warning: HostAliases parameter is not defined, $PROG choose \"$SiteDomain localhost 127.0.0.1\"."); push @HostAliases,qr/^$sitetoanalyze$/i; push @HostAliases,qr/^localhost$/i; push @HostAliases,qr/^127\.0\.0\.1$/i; } else { unshift @HostAliases,qr/^$sitetoanalyze$/i; } # Add SiteDomain as first value # Optimize arrays @HostAliases=&OptimizeArray(\@HostAliases,1); if ($Debug) { debug("HostAliases precompiled regex list is now @HostAliases",1); } @SkipDNSLookupFor=&OptimizeArray(\@SkipDNSLookupFor,1); if ($Debug) { debug("SkipDNSLookupFor precompiled regex list is now @SkipDNSLookupFor",1); } @SkipHosts=&OptimizeArray(\@SkipHosts,1); if ($Debug) { debug("SkipHosts precompiled regex list is now @SkipHosts",1); } @SkipUserAgents=&OptimizeArray(\@SkipUserAgents,1); if ($Debug) { debug("SkipUserAgents precompiled regex list is now @SkipUserAgents",1); } @SkipFiles=&OptimizeArray(\@SkipFiles,$URLNotCaseSensitive); if ($Debug) { debug("SkipFiles precompiled regex list is now @SkipFiles",1); } @OnlyHosts=&OptimizeArray(\@OnlyHosts,1); if ($Debug) { debug("OnlyHosts precompiled regex list is now @OnlyHosts",1); } @OnlyUserAgents=&OptimizeArray(\@OnlyUserAgents,1); if ($Debug) { debug("OnlyUserAgents precompiled regex list is now @OnlyUserAgents",1); } #Params OnlyXXX defined by E-Lane @OnlyUsers=&OptimizeArray(\@OnlyUsers,1); if ($Debug) { debug("OnlyUsers precompiled regex list is now @OnlyUsers",1); } @OnlyLines=&OptimizeArray(\@OnlyLines,1); if ($Debug) { debug("OnlyLines precompiled regex list is now @OnlyLines",1); } @OnlyFiles=&OptimizeArray(\@OnlyFiles,$URLNotCaseSensitive); if ($Debug) { debug("OnlyFiles precompiled regex list is now @OnlyFiles",1); } # Precompile the regex search strings with qr @RobotsSearchIDOrder=map{qr/$_/i} @RobotsSearchIDOrder; @WormsSearchIDOrder=map{qr/$_/i} @WormsSearchIDOrder; @BrowsersSearchIDOrder=map{qr/$_/i} @BrowsersSearchIDOrder; @OSSearchIDOrder=map{qr/$_/i} @OSSearchIDOrder; @SearchEnginesSearchIDOrder=map{qr/$_/i} @SearchEnginesSearchIDOrder; my $miscquoted=quotemeta("$MiscTrackerUrl"); my $defquoted=quotemeta("/$DefaultFile[0]"); my $sitewithoutwww=lc($SiteDomain); $sitewithoutwww =~ s/www\.//; $sitewithoutwww=quotemeta($sitewithoutwww); # Define precompiled regex my $regmisc=qr/^$miscquoted/; my $regfavico=qr/\/favicon\.ico$/i; my $regrobot=qr/^\/robots\.txt$/i; my $regtruncanchor=qr/#(\w*)$/; my $regtruncurl=qr/([$URLQuerySeparators])(.*)$/; my $regext=qr/\.(\w{1,6})$/; my $regdefault; if ($URLNotCaseSensitive) { $regdefault=qr/$defquoted$/i; } else { $regdefault=qr/$defquoted$/; } my $regipv4=qr/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/; my $regipv6=qr/^[0-9A-F]*:/i; my $regvermsie=qr/msie([+_ ]|)([\d\.]*)/i; my $regvernetscape=qr/netscape.?\/([\d\.]*)/i; my $regvermozilla=qr/mozilla(\/|)([\d\.]*)/i; my $regnotie=qr/webtv|omniweb|opera/i; my $regnotnetscape=qr/gecko|compatible|opera|galeon|safari/i; my $regreferer=qr/^(\w+):\/\/([^\/:]+)(:\d+|)/; my $regreferernoquery=qr/^([^$URLQuerySeparators]+)/; my $reglocal=qr/^(www\.|)$sitewithoutwww/i; my $regget=qr/get/i; my $regsent=qr/sent/i; my $regput=qr/put/i; # Define value of $PerlParsingFormat and @fieldlib &DefinePerlParsingFormat(); # Load DNS Cache Files #------------------------------------------ if ($DNSLookup) { &Read_DNS_Cache(\%MyDNSTable,"$DNSStaticCacheFile","",1); # Load with save into a second plugin file if plugin enabled and second file not up to date. No use of FileSuffix if ($DNSLookup == 1) { # System DNS lookup required #if (! eval("use Socket;")) { error("Failed to load perl module Socket."); } #use Socket; &Read_DNS_Cache(\%TmpDNSLookup,"$DNSLastUpdateCacheFile","$FileSuffix",0); # Load with no save into a second plugin file. Use FileSuffix } } # Processing log #------------------------------------------ if ($EnableLockForUpdate) { # Trap signals to remove lock $SIG{INT} = \&SigHandler; # 2 #$SIG{KILL} = \&SigHandler; # 9 #$SIG{TERM} = \&SigHandler; # 15 # Set AWStats update lock &Lock_Update(1); } if ($Debug) { debug("Start Update process (lastprocessedmonth=$lastprocessedmonth, lastprocessedyear=$lastprocessedyear)"); } # Open log file if ($Debug) { debug("Open log file \"$LogFile\""); } open(LOG,"$LogFile") || error("Couldn't open server log file \"$LogFile\" : $!"); binmode LOG; # Avoid premature EOF due to log files corrupted with \cZ or bin chars # Define local variables for loop scan my @field=(); my $counterforflushtest=0; my $qualifdrop=''; my $countedtraffic=0; # Reset chrono for benchmark (first call to GetDelaySinceStart) &GetDelaySinceStart(1); if (! scalar keys %HTMLOutput) { print "Phase 1 : First bypass old records, searching new record...\n"; } # Can we try a direct seek access in log ? my $line; if ($LastLine && $LastLineNumber && $LastLineOffset && $LastLineChecksum) { # Try a direct seek access to save time if ($Debug) { debug("Try a direct access to LastLine=$LastLine, LastLineNumber=$LastLineNumber, LastLineOffset=$LastLineOffset, LastLineChecksum=$LastLineChecksum"); } seek(LOG,$LastLineOffset,0); if ($line=) { chomp $line; $line =~ s/\r$//; @field=map(/$PerlParsingFormat/,$line); if ($Debug) { my $string=''; foreach (0..@field-1) { $string.="$fieldlib[$_]=$field[$_] "; } if ($Debug) { debug(" Read line after direct access: $string",1); } } my $checksum=&CheckSum($line); if ($Debug) { debug(" LastLineChecksum=$LastLineChecksum, Read line checksum=$checksum",1); } if ($checksum == $LastLineChecksum ) { if (! scalar keys %HTMLOutput) { print "Direct access after last parsed record (after line $LastLineNumber)\n"; } $lastlinenb=$LastLineNumber; $lastlineoffset=$LastLineOffset; $lastlineoffsetnext=tell LOG; $NewLinePhase=1; } else { if (! scalar keys %HTMLOutput) { print "Direct access to last remembered record has fallen on another record.\nSo searching new records from beginning of log file...\n"; } $lastlinenb=0; $lastlineoffset=0; $lastlineoffsetnext=0; seek(LOG,0,0); } } else { if (! scalar keys %HTMLOutput) { print "Direct access to last remembered record is out of file.\nSo searching it from beginning of log file...\n"; } $lastlinenb=0; $lastlineoffset=0; $lastlineoffsetnext=0; seek(LOG,0,0); } } else { # No try of direct seek access if (! scalar keys %HTMLOutput) { print "Searching new records from beginning of log file...\n"; } $lastlinenb=0; $lastlineoffset=0; $lastlineoffsetnext=0; } $lastdate = 0; $position = 0; $counter = 1; while ($line=) { chomp $line; $line =~ s/\r$//; if ($UpdateFor && $NbOfLinesParsed >= $UpdateFor) { last; } $NbOfLinesParsed++; $lastlineoffset=$lastlineoffsetnext; $lastlineoffsetnext=tell LOG; if ($ShowSteps) { if ((++$NbOfLinesShowsteps & $NBOFLINESFORBENCHMARK) == 0) { my $delay=&GetDelaySinceStart(0); print "$NbOfLinesParsed lines processed (".($delay>0?$delay:1000)." ms, ".int(1000*$NbOfLinesShowsteps/($delay>0?$delay:1000))." lines/second)\n"; } } # Parse line record to get all required fields if (! (@field=map(/$PerlParsingFormat/,$line))) { $NbOfLinesCorrupted++; if ($ShowCorrupted) { if ($line =~ /^#/ || $line =~ /^!/) { print "Corrupted record line ".($lastlinenb+$NbOfLinesParsed)." (comment line): $line\n"; } elsif ($line =~ /^\s*$/) { print "Corrupted record line ".($lastlinenb+$NbOfLinesParsed)." (blank line)\n"; } else { print "Corrupted record line ".($lastlinenb+$NbOfLinesParsed)." (record format does not match LogFormat parameter): $line\n"; } } if ($NbOfLinesParsed >= $NbOfLinesForCorruptedLog && $NbOfLinesParsed == $NbOfLinesCorrupted) { error("Format error",$line,$LogFile); } # Exit with format error if ($line =~ /^__end_of_file__/i) { last; } # For test purpose only next; } if ($Debug) { my $string=''; foreach (0..@field-1) { $string.="$fieldlib[$_]=$field[$_] "; } if ($Debug) { debug(" Correct format line ".($lastlinenb+$NbOfLinesParsed).": $string",4); } } # Drop wrong virtual host name #---------------------------------------------------------------------- if ($pos_vh>=0 && $field[$pos_vh] !~ /^$SiteDomain$/i) { my $skip=1; foreach (@HostAliases) { if ($field[$pos_vh] =~ /$_/) { $skip=0; last; } } if ($skip) { $NbOfLinesDropped++; if ($ShowDropped) { print "Dropped record (virtual hostname '$field[$pos_vh]' does not match SiteDomain='$SiteDomain' nor HostAliases parameters): $line\n"; } next; } } # Drop wrong method/protocol #--------------------------- if ($LogType ne 'M') { $field[$pos_url] =~ s/\s/%20/g; } if ($LogType eq 'W' && ($field[$pos_method] eq 'GET' || $field[$pos_method] eq 'POST' || $field[$pos_method] eq 'HEAD' || $field[$pos_method] =~ /OK/i || $field[$pos_method] =~ /ERR\!/i)) { # HTTP request. Keep only GET, POST, HEAD, *OK* and ERR! for Webstar. Do not keep OPTIONS, TRACE } elsif (($LogType eq 'W' || $LogType eq 'S') && ($field[$pos_method] eq 'GET' || $field[$pos_method] eq 'mms' || $field[$pos_method] eq 'rtsp' || $field[$pos_method] eq 'http' || $field[$pos_method] eq 'RTP')) { # Streaming request (windows media server, realmedia or darwin streaming server) } elsif ($LogType eq 'M' && $field[$pos_method] eq 'SMTP') { # Mail request ('SMTP' for mail log with maillogconvert.pl preprocessor) } elsif ($LogType eq 'F' && ($field[$pos_method] eq 'RETR' || $field[$pos_method] eq 'o' || $field[$pos_method] =~ /$regget/o)) { # FTP GET request } elsif ($LogType eq 'F' && ($field[$pos_method] eq 'STOR' || $field[$pos_method] eq 'i' || $field[$pos_method] =~ /$regsent/o || $field[$pos_method] =~ /$regput/o)) { # FTP SENT request } else { $NbOfLinesDropped++; if ($ShowDropped) { print "Dropped record (method/protocol '$field[$pos_method]' not qualified when LogType=$LogType): $line\n"; } next; } $field[$pos_date] =~ tr/,-\/ \t/:::::/s; # " \t" is used instead of "\s" not known with tr my @dateparts=split(/:/,$field[$pos_date]); # tr and split faster than @dateparts=split(/[\/\-:\s]/,$field[$pos_date]) # Detected date format: dddddddddd, YYYY-MM-DD HH:MM:SS (IIS), MM/DD/YY\tHH:MM:SS, # DD/Month/YYYY:HH:MM:SS (Apache), DD/MM/YYYY HH:MM:SS, Mon DD HH:MM:SS if (! $dateparts[1]) { # Unix timestamp ($dateparts[5],$dateparts[4],$dateparts[3],$dateparts[0],$dateparts[1],$dateparts[2]) = localtime(int($field[$pos_date])); $dateparts[1]++;$dateparts[2]+=1900; } elsif ($dateparts[0] =~ /^....$/) { my $tmp=$dateparts[0]; $dateparts[0]=$dateparts[2]; $dateparts[2]=$tmp; } elsif ($field[$pos_date] =~ /^..:..:..:/) { $dateparts[2]+=2000; my $tmp=$dateparts[0]; $dateparts[0]=$dateparts[1]; $dateparts[1]=$tmp; } elsif ($dateparts[0] =~ /^...$/) { my $tmp=$dateparts[0]; $dateparts[0]=$dateparts[1]; $dateparts[1]=$tmp; $tmp=$dateparts[5]; $dateparts[5]=$dateparts[4]; $dateparts[4]=$dateparts[3]; $dateparts[3]=$dateparts[2]; $dateparts[2]=$tmp||$nowyear; } if ($MonthNum{$dateparts[1]}) { $dateparts[1]=$MonthNum{$dateparts[1]}; } # Change lib month in num month if necessary # Now @dateparts is (DD,MM,YYYY,HH,MM,SS) and we're going to create $timerecord=YYYYMMDDHHMMSS # Plugin call : Convert a @datepart into another @datepart if ($PluginsLoaded{'ChangeTime'}{'timezone'}) { @dateparts=ChangeTime_timezone(\@dateparts); } my $yearrecord=int($dateparts[2]); my $monthrecord=int($dateparts[1]); my $hourrecord=int($dateparts[3]); my $yearmonthdayrecord=sprintf("$dateparts[2]%02i%02i",$dateparts[1],$dateparts[0]); my $timerecord=((int("$yearmonthdayrecord")*100+$dateparts[3])*100+$dateparts[4])*100+$dateparts[5]; # E-lane, calculating moments in which more hits are produced $date = $timerecord; if($lastdate == 0){ $counter = 0; } if($lastdate == $date){ $counter++; } else{ if($position < 10){ if($counter > 0){ @request[$position] = $counter; @date_request[$position] = $lastdate; $position++; } } else{ if($position == 10){ $counterSmall = @request[0]; $positionSmall = 0; $i=0; foreach (@request){ if($counterSmall > @request[$i]){ $counterSmall = @request[$i]; $positionSmall = $i; } $i++; } if(@request[$positionSmall] < $counter){ @request[$positionSmall] = $counter; @date_request[$positionSmall] = $lastdate; } } } $counter = 1; $lastdate = $date; } #E-lane end # Check date #----------------------- if ($LogType eq 'M' && $timerecord > $tomorrowtime) { # Postfix/Sendmail does not store year, so we assume that year is year-1 if record is in future $yearrecord--; $yearmonthdayrecord=sprintf("$yearrecord%02i%02i",$dateparts[1],$dateparts[0]); $timerecord=((int("$yearmonthdayrecord")*100+$dateparts[3])*100+$dateparts[4])*100+$dateparts[5]; } if ($timerecord < 10000000000000 || $timerecord > $tomorrowtime) { $NbOfLinesCorrupted++; if ($ShowCorrupted) { print "Corrupted record (invalid date, timerecord=$timerecord): $line\n"; } next; # Should not happen, kept in case of parasite/corrupted line } if ($NewLinePhase) { # TODO NOTSORTEDRECORDTOLERANCE does not work around midnight if ($timerecord < ($LastLine - $NOTSORTEDRECORDTOLERANCE)) { # Should not happen, kept in case of parasite/corrupted old line $NbOfLinesCorrupted++; if ($ShowCorrupted) { print "Corrupted record (date $timerecord lower than $LastLine-$NOTSORTEDRECORDTOLERANCE): $line\n"; } next; } } else { if ($timerecord <= $LastLine) { # Already processed $NbOfOldLines++; next; } # We found a new line. This will replace comparison "<=" with "<" between timerecord and LastLine (we should have only new lines now) $NewLinePhase=1; # We will never enter here again if ($ShowSteps) { if ($NbOfLinesShowsteps > 1 && ($NbOfLinesShowsteps & $NBOFLINESFORBENCHMARK)) { my $delay=&GetDelaySinceStart(0); print "".($NbOfLinesParsed-1)." lines processed (".($delay>0?$delay:1000)." ms, ".int(1000*($NbOfLinesShowsteps-1)/($delay>0?$delay:1000))." lines/second)\n"; } &GetDelaySinceStart(1); $NbOfLinesShowsteps=1; } if (! scalar keys %HTMLOutput) { print "Phase 2 : Now process new records (Flush history on disk after ".($LIMITFLUSH<<2)." hosts)...\n"; #print "Phase 2 : Now process new records (Flush history on disk after ".($LIMITFLUSH<<2)." hosts or ".($LIMITFLUSH)." URLs)...\n"; } } # Convert URL for Webstar to common URL if ($LogFormat eq '3') { $field[$pos_url]=~s/:/\//g; if ($field[$pos_code] eq '-') { $field[$pos_code]='200'; } } # Here, field array, timerecord and yearmonthdayrecord are initialized for log record if ($Debug) { debug(" This is a not already processed record ($timerecord)",4); } # We found a new line #---------------------------------------- if ($timerecord > $LastLine) { $LastLine = $timerecord; } # Test should always be true except with not sorted log files # Skip for some client host IP addresses, some URLs, other URLs if (@SkipHosts && (&SkipHost($field[$pos_host]) || ($pos_hostr && &SkipHost($field[$pos_hostr])))) { $qualifdrop="Dropped record (host $field[$pos_host]".($pos_hostr?" and $field[$pos_hostr]":"")." not qualified by SkipHosts)"; } elsif (@SkipFiles && &SkipFile($field[$pos_url])) { $qualifdrop="Dropped record (URL $field[$pos_url] not qualified by SkipFiles)"; } elsif (@SkipUserAgents && $pos_agent >= 0 && &SkipUserAgent($field[$pos_agent])) { $qualifdrop="Dropped record (user agent '$field[$pos_agent]' not qualified by SkipUserAgents)"; } elsif (@OnlyHosts && ! &OnlyHost($field[$pos_host]) && (! $pos_hostr || ! &OnlyHost($field[$pos_hostr]))) { $qualifdrop="Dropped record (host $field[$pos_host]".($pos_hostr?" and $field[$pos_hostr]":"")." not qualified by OnlyHosts)"; } elsif (@OnlyFiles && ! &OnlyFile($field[$pos_url])) { $qualifdrop="Dropped record (URL $field[$pos_url] not qualified by OnlyFiles)"; } elsif (@OnlyUserAgents && ! &OnlyUserAgent($field[$pos_agent])) { $qualifdrop="Dropped record (user agent '$field[$pos_agent]' not qualified by OnlyUserAgents)"; } #Params OnlyXXX defined by E-Lane elsif (@OnlyUsers && ! &OnlyUser($field[$pos_logname])) { $qualifdrop="Used record (user '$field[$pos_logname]' qualified by OnlyUsers)"; } elsif (@OnlyLines && ! &OnlyLine($line)) { $qualifdrop="Used line ('$field[$pos_logname]' qualified by OnlyLines)"; } if ($qualifdrop) { $NbOfLinesDropped++; if ($Debug) { debug("$qualifdrop: $line",4); } if ($ShowDropped) { print "$qualifdrop: $line\n"; } $qualifdrop=''; next; } # Record is approved #------------------- # Is it in a new month section ? #------------------------------- if ((($monthrecord > $lastprocessedmonth) && ($yearrecord >= $lastprocessedyear)) || ($yearrecord > $lastprocessedyear)) { # A new month to process if ($lastprocessedmonth) { # We save data of processed month &Read_History_With_TmpUpdate($lastprocessedyear,$lastprocessedmonth,1,1,"all",($lastlinenb+$NbOfLinesParsed),$lastlineoffset,&CheckSum($line)); # &Read_History_With_TmpUpdate($YearRequired,$MonthRequired,1,1,"all",($lastlinenb+$NbOfLinesParsed),$lastlineoffset,&CheckSum($line)); $counterforflushtest=0; # We reset counterforflushtest } $lastprocessedyearmonth=sprintf("%04i%02i",$lastprocessedyear=$yearrecord,$lastprocessedmonth=$monthrecord); } $countedtraffic=0; $NbOfNewLines++; # Convert $field[$pos_size] # if ($field[$pos_size] eq '-') { $field[$pos_size]=0; } # Define a clean target URL and referrer URL # We keep a clean $field[$pos_url] and # we store original value for urlwithnoquery, tokenquery and standalonequery #--------------------------------------------------------------------------- if ($URLNotCaseSensitive) { $field[$pos_url]=lc($field[$pos_url]); } # Possible URL syntax for $field[$pos_url]: /mydir/mypage.ext?param1=x¶m2=y#aaa, /mydir/mypage.ext#aaa, / my $urlwithnoquery; my $tokenquery; my $standalonequery; my $anchor=''; if ($field[$pos_url] =~ s/$regtruncanchor//o) { $anchor=$1; } # Remove and save anchor if ($URLWithQuery) { $urlwithnoquery=$field[$pos_url]; my $foundparam=($urlwithnoquery =~ s/$regtruncurl//o); $tokenquery=$1||''; $standalonequery=$2||''; # For IIS setup, if pos_query is enabled we need to combine the URL to query strings if (! $foundparam && $pos_query >=0 && $field[$pos_query] && $field[$pos_query] ne '-') { $foundparam=1; $tokenquery='?'; $standalonequery=$field[$pos_query]; # Define query $field[$pos_url] .= '?'.$field[$pos_query]; } if ($foundparam) { # Keep only params that are defined in URLWithQueryWithOnlyFollowingParameters my $newstandalonequery=''; if (@URLWithQueryWithOnly) { foreach (@URLWithQueryWithOnly) { foreach my $p (split(/&/,$standalonequery)) { if ($URLNotCaseSensitive) { if ($p =~ /^$_=/i) { $newstandalonequery.="$p&"; last; } } else { if ($p =~ /^$_=/) { $newstandalonequery.="$p&"; last; } } } } chop $newstandalonequery; } # Remove params that are marked to be ignored in URLWithQueryWithoutFollowingParameters elsif (@URLWithQueryWithout) { foreach my $p (split(/&/,$standalonequery)) { my $found=0; foreach (@URLWithQueryWithout) { #if ($Debug) { debug(" Check if '$_=' is param '$p' to remove it from query",5); } if ($URLNotCaseSensitive) { if ($p =~ /^$_=/i) { $found=1; last; } } else { if ($p =~ /^$_=/) { $found=1; last; } } } if (! $found) { $newstandalonequery.="$p&"; } } chop $newstandalonequery; } else { $newstandalonequery=$standalonequery; } # Define query $field[$pos_url]=$urlwithnoquery; if ($newstandalonequery) { $field[$pos_url].="$tokenquery$newstandalonequery"; } } } else { # Trunc parameters of URL $field[$pos_url] =~ s/$regtruncurl//o; $urlwithnoquery=$field[$pos_url]; $tokenquery=$1||''; $standalonequery=$2||''; # For IIS setup, if pos_query is enabled we need to use it for query strings if ($pos_query >=0 && $field[$pos_query] && $field[$pos_query] ne '-') { $tokenquery='?'; $standalonequery=$field[$pos_query]; } } if ($URLWithAnchor && $anchor) { $field[$pos_url].="#$anchor"; } # Restore anchor # Here now urlwithnoquery is /mydir/mypage.ext, /mydir, /, /page#XXX # Here now tokenquery is '' or '?' or ';' # Here now standalonequery is '' or 'param1=x' # Define page and extension #-------------------------- my $PageBool=1; # Extension my $extension; if ($urlwithnoquery =~ /$regext/o || ($urlwithnoquery =~ /[\\\/]$/ && $DefaultFile[0] =~ /$regext/o)) { $extension=($LevelForFileTypesDetection>=2 || $MimeHashFamily{$1})?lc($1):'Unknown'; if ($NotPageList{$extension}) { $PageBool=0; } } else { $extension='Unknown'; } # Analyze: misc tracker (must be before return code) #--------------------------------------------------- if ($urlwithnoquery =~ /$regmisc/o) { if ($Debug) { debug(" Found an URL that is a MiscTracker record with standalonequery=$standalonequery",2); } my $foundparam=0; foreach (split(/&/,$standalonequery)) { if ($_ =~ /^screen=(\d+)x(\d+)/i) { $foundparam++; $_screensize_h{"$1x$2"}++; next; } #if ($_ =~ /cdi=(\d+)/i) { $foundparam++; $_screendepth_h{"$1"}++; next; } if ($_ =~ /^nojs=(\w+)/i) { $foundparam++; if ($1 eq 'y') { $_misc_h{"JavascriptDisabled"}++; } next; } if ($_ =~ /^java=(\w+)/i) { $foundparam++; if ($1 eq 'true') { $_misc_h{"JavaEnabled"}++; } next; } if ($_ =~ /^shk=(\w+)/i) { $foundparam++; if ($1 eq 'y') { $_misc_h{"DirectorSupport"}++; } next; } if ($_ =~ /^fla=(\w+)/i) { $foundparam++; if ($1 eq 'y') { $_misc_h{"FlashSupport"}++; } next; } if ($_ =~ /^rp=(\w+)/i) { $foundparam++; if ($1 eq 'y') { $_misc_h{"RealPlayerSupport"}++; } next; } if ($_ =~ /^mov=(\w+)/i) { $foundparam++; if ($1 eq 'y') { $_misc_h{"QuickTimeSupport"}++; } next; } if ($_ =~ /^wma=(\w+)/i) { $foundparam++; if ($1 eq 'y') { $_misc_h{"WindowsMediaPlayerSupport"}++; } next; } if ($_ =~ /^pdf=(\w+)/i) { $foundparam++; if ($1 eq 'y') { $_misc_h{"PDFSupport"}++; } next; } } if ($foundparam) { $_misc_h{"TotalMisc"}++; } } # Analyze: favicon (countedtraffic=>1) #------------------------------------- if ($pos_referer >= 0 && $field[$pos_referer] && $urlwithnoquery =~ /$regfavico/o) { if (($field[$pos_code] != 404 || $urlwithnoquery !~ /\/.+\/favicon\.ico$/i) && ($field[$pos_agent] =~ /MSIE/)) { # We don't count one hit if (not on root and error) and MSIE # If error not on root, another hit will be made on root. If not MSIE, hit are made not only for "Adding". $_misc_h{'AddToFavourites'}++; # Hit on favicon on root or without error, we count it } $countedtraffic=1; # favicon is the only case not counted anywhere } # Analyze: Worms (countedtraffic=>1 if worm) #------------------------------------------- if (! $countedtraffic) { if ($LevelForWormsDetection) { foreach (@WormsSearchIDOrder) { if ($field[$pos_url] =~ /$_/) { # It's a worm my $worm=&UnCompileRegex($_); if ($Debug) { debug(" Record is a hit from a worm identified by '$worm'",2); } $worm=$WormsHashID{$worm}||'unknown'; $_worm_h{$worm}++; $_worm_k{$worm}+=int($field[$pos_size]); $_worm_l{$worm}=$timerecord; $countedtraffic=1; if ($PageBool) { $_time_nv_p[$hourrecord]++; } $_time_nv_h[$hourrecord]++; $_time_nv_k[$hourrecord]+=int($field[$pos_size]); last; } } } } # Analyze: Status code (countedtraffic=>1 if error) #-------------------------------------------------- if (! $countedtraffic) { if ($LogType eq 'W' || $LogType eq 'S') { # HTTP record or Stream record if ($ValidHTTPCodes{$field[$pos_code]}) { # Code is valid if ($field[$pos_code] == 304) { $field[$pos_size]=0; } } else { # Code is not valid if ($field[$pos_code] !~ /^\d\d\d$/) { $field[$pos_code]=999; } $_errors_h{$field[$pos_code]}++; $_errors_k{$field[$pos_code]}+=int($field[$pos_size]); foreach my $code (keys %TrapInfosForHTTPErrorCodes) { if ($field[$pos_code] == $code) { # This is an error code which referrer need to be tracked my $newurl=substr($field[$pos_url],0,$MaxLengthOfStoredURL); $newurl =~ s/[$URLQuerySeparators].*$//; $_sider404_h{$newurl}++; my $newreferer=$field[$pos_referer]; if (! $URLReferrerWithQuery) { $newreferer =~ s/[$URLQuerySeparators].*$//; } $_referer404_h{$newurl}=$newreferer; last; } } if ($Debug) { debug(" Record stored in the status code chart (status code=$field[$pos_code])",2); } $countedtraffic=1; if ($PageBool) { $_time_nv_p[$hourrecord]++; } $_time_nv_h[$hourrecord]++; $_time_nv_k[$hourrecord]+=int($field[$pos_size]); } } elsif ($LogType eq 'M') { # Mail record if (! $ValidSMTPCodes{$field[$pos_code]}) { # Code is not valid $_errors_h{$field[$pos_code]}++; $_errors_k{$field[$pos_code]}+=int($field[$pos_size]); # Size is often 0 when error if ($Debug) { debug(" Record stored in the status code chart (status code=$field[$pos_code])",2); } $countedtraffic=1; if ($PageBool) { $_time_nv_p[$hourrecord]++; } $_time_nv_h[$hourrecord]++; $_time_nv_k[$hourrecord]+=int($field[$pos_size]); } } elsif ($LogType eq 'F') { # FTP record } } # Analyze: Robot from robot database (countedtraffic=>1 if robot) #---------------------------------------------------------------- if (! $countedtraffic) { if ($pos_agent >= 0) { if ($DecodeUA) { $field[$pos_agent] =~ s/%20/_/g; } # This is to support servers (like Roxen) that writes user agent with %20 in it $UserAgent=$field[$pos_agent]; if ($LevelForRobotsDetection) { my $uarobot=$TmpRobot{$UserAgent}; if (! $uarobot) { #study $UserAgent; Does not increase speed foreach (@RobotsSearchIDOrder) { if ($UserAgent =~ /$_/) { my $bot=&UnCompileRegex($_); $TmpRobot{$UserAgent}=$uarobot="$bot"; # Last time, we won't search if robot or not. We know it is. if ($Debug) { debug(" UserAgent '$UserAgent' is added to TmpRobot with value '$bot'",2); } last; } } if (! $uarobot) { # Last time, we won't search if robot or not. We know it's not. $TmpRobot{$UserAgent}=$uarobot='-'; } } if ($uarobot ne '-') { # If robot, we stop here if ($Debug) { debug(" UserAgent '$UserAgent' contains robot ID '$uarobot'",2); } $_robot_h{$uarobot}++; $_robot_k{$uarobot}+=int($field[$pos_size]); $_robot_l{$uarobot}=$timerecord; if ($urlwithnoquery =~ /$regrobot/o) { $_robot_r{$uarobot}++; } $countedtraffic=1; if ($PageBool) { $_time_nv_p[$hourrecord]++; } $_time_nv_h[$hourrecord]++; $_time_nv_k[$hourrecord]+=int($field[$pos_size]); } } } } # Analyze: Robot from "hit on robots.txt" file (countedtraffic=>1 if robot) # ------------------------------------------------------------------------- if (! $countedtraffic) { if ($urlwithnoquery =~ /$regrobot/o) { if ($Debug) { debug(" It's an unknown robot",2); } $_robot_h{'unknown'}++; $_robot_k{'unknown'}+=int($field[$pos_size]); $_robot_l{'unknown'}=$timerecord; $_robot_r{'unknown'}++; $countedtraffic=1; if ($PageBool) { $_time_nv_p[$hourrecord]++; } $_time_nv_h[$hourrecord]++; $_time_nv_k[$hourrecord]+=int($field[$pos_size]); } } # Analyze: File type - Compression #--------------------------------- if (! $countedtraffic) { if ($LevelForFileTypesDetection) { $_filetypes_h{$extension}++; $_filetypes_k{$extension}+=int($field[$pos_size]); # TODO can cause a warning # Compression if ($pos_gzipin>=0 && $field[$pos_gzipin]) { # If in and out in log my ($notused,$in)=split(/:/,$field[$pos_gzipin]); my ($notused1,$out,$notused2)=split(/:/,$field[$pos_gzipout]); if ($out) { $_filetypes_gz_in{$extension}+=$in; $_filetypes_gz_out{$extension}+=$out; } } elsif ($pos_compratio>=0 && ($field[$pos_compratio] =~ /(\d+)/)) { # Calculate in/out size from percentage if ($fieldlib[$pos_compratio] eq 'gzipratio') { # with mod_gzip: % is size (before-after)/before (low for jpg) ?????????? $_filetypes_gz_in{$extension}+=int($field[$pos_size]*100/((100-$1)||1)); } else { # with mod_deflate: % is size after/before (high for jpg) $_filetypes_gz_in{$extension}+=int($field[$pos_size]*100/($1||1)); } $_filetypes_gz_out{$extension}+=int($field[$pos_size]); } } # Analyze: Date - Hour - Pages - Hits - Kilo #------------------------------------------- if ($PageBool) { # Replace default page name with / only ('if' is to increase speed when only 1 value in @DefaultFile) if (@DefaultFile > 1) { foreach my $elem (@DefaultFile) { if ($field[$pos_url] =~ s/\/$elem$/\//o) { last; } } } else { $field[$pos_url] =~ s/$regdefault/\//o; } # FirstTime and LastTime are First and Last human visits (so changed if access to a page) $FirstTime{$lastprocessedyearmonth}||=$timerecord; $LastTime{$lastprocessedyearmonth}=$timerecord; $DayPages{$yearmonthdayrecord}++; $_url_p{$field[$pos_url]}++; #Count accesses for page (page) $_url_k{$field[$pos_url]}+=int($field[$pos_size]); $_time_p[$hourrecord]++; #Count accesses for hour (page) # TODO Use an id for hash key of url # $_url_t{$_url_id} } $_time_h[$hourrecord]++; $_time_k[$hourrecord]+=int($field[$pos_size]); $DayHits{$yearmonthdayrecord}++; #Count accesses for hour (hit) $DayBytes{$yearmonthdayrecord}+=int($field[$pos_size]); #Count accesses for hour (kb) # Analyze: Login #--------------- if ($pos_logname>=0 && $field[$pos_logname] && $field[$pos_logname] ne '-') { $field[$pos_logname] =~ s/ /_/g; # This is to allow space in logname if ($LogFormat eq '6') { $field[$pos_logname] =~ s/^\"//; $field[$pos_logname] =~ s/\"$//;} # logname field has " with Domino 6+ if ($AuthenticatedUsersNotCaseSensitive) { $field[$pos_logname]=lc($field[$pos_logname]); } # We found an authenticated user if ($PageBool) { $_login_p{$field[$pos_logname]}++; } #Count accesses for page (page) $_login_h{$field[$pos_logname]}++; #Count accesses for page (hit) $_login_k{$field[$pos_logname]}+=int($field[$pos_size]); #Count accesses for page (kb) $_login_l{$field[$pos_logname]}=$timerecord; } } # Do DNS lookup #-------------- my $Host=$field[$pos_host]; my $HostResolved=''; if (! $countedtraffic) { my $ip=0; if ($DNSLookup) { # DNS lookup is 1 or 2 if ($Host =~ /$regipv4/o) { $ip=4; } # IPv4 elsif ($Host =~ /$regipv6/o) { $ip=6; } # IPv6 if ($ip) { # Check in static DNS cache file $HostResolved=$MyDNSTable{$Host}; if ($HostResolved) { if ($Debug) { debug(" DNS lookup asked for $Host and found in static DNS cache file: $HostResolved",4); } } elsif ($DNSLookup==1) { # Check in session cache (dynamic DNS cache file + session DNS cache) $HostResolved=$TmpDNSLookup{$Host}; if (! $HostResolved) { if (@SkipDNSLookupFor && &SkipDNSLookup($Host)) { $HostResolved=$TmpDNSLookup{$Host}='*'; if ($Debug) { debug(" No need of reverse DNS lookup for $Host, skipped at user request.",4); } } else { if ($ip == 4) { my $lookupresult=gethostbyaddr(pack("C4",split(/\./,$Host)),AF_INET); # This is very slow, may spend 20 seconds if (! $lookupresult || $lookupresult =~ /$regipv4/o || ! IsAscii($lookupresult)) { $TmpDNSLookup{$Host}=$HostResolved='*'; } else { $TmpDNSLookup{$Host}=$HostResolved=$lookupresult; } if ($Debug) { debug(" Reverse DNS lookup for $Host done: $HostResolved",4); } } elsif ($ip == 6) { if ($PluginsLoaded{'GetResolvedIP'}{'ipv6'}) { my $lookupresult=GetResolvedIP_ipv6($Host); if (! $lookupresult || ! IsAscii($lookupresult)) { $TmpDNSLookup{$Host}=$HostResolved='*'; } else { $TmpDNSLookup{$Host}=$HostResolved=$lookupresult; } } else { $TmpDNSLookup{$Host}=$HostResolved='*'; warning("Reverse DNS lookup for $Host not available without ipv6 plugin enabled."); } } else { error("Bad value vor ip"); } } } } else { $HostResolved='*'; if ($Debug) { debug(" DNS lookup by static DNS cache file asked for $Host but not found.",4); } } } else { if ($Debug) { debug(" DNS lookup asked for $Host but this is not an IP address.",4); } $DNSLookupAlreadyDone=$LogFile; } } else { if ($Host =~ /$regipv4/o) { $HostResolved='*'; $ip=4; } # IPv4 elsif ($Host =~ /$regipv6/o) { $HostResolved='*'; $ip=6; } # IPv6 if ($Debug) { debug(" No DNS lookup asked.",4); } } # Analyze: Country (Top-level domain) #------------------------------------ if ($Debug) { debug(" Search country (Host=$Host HostResolved=$HostResolved ip=$ip)",4); } my $Domain='ip'; # Set $HostResolved to host and resolve domain if ($HostResolved eq '*') { # $Host is an IP address and is not resolved (failed or not asked) or resolution gives an IP address $HostResolved = $Host; # Resolve Domain if ($PluginsLoaded{'GetCountryCodeByAddr'}{'geoipfree'}) { $Domain=GetCountryCodeByAddr_geoipfree($HostResolved); } elsif ($PluginsLoaded{'GetCountryCodeByAddr'}{'geoip'}) { $Domain=GetCountryCodeByAddr_geoip($HostResolved); } if ($AtLeastOneSectionPlugin) { foreach my $pluginname (keys %{$PluginsLoaded{'SectionProcessHost'}}) { my $function="SectionProcessHost_$pluginname(\$HostResolved)"; eval("$function"); } } } else { # $Host was already a host name ($ip=0, $Host=name, $HostResolved='') or has been resolved ($ip>0, $Host=ip, $HostResolved defined) $HostResolved = lc($HostResolved?$HostResolved:$Host); # Resolve Domain if ($ip) { # If we have ip, we use it in priority instead of hostname if ($PluginsLoaded{'GetCountryCodeByAddr'}{'geoipfree'}) { $Domain=GetCountryCodeByAddr_geoipfree($Host); } elsif ($PluginsLoaded{'GetCountryCodeByAddr'}{'geoip'}) { $Domain=GetCountryCodeByAddr_geoip($Host); } elsif ($HostResolved =~ /\.(\w+)$/) { $Domain=$1; } if ($AtLeastOneSectionPlugin) { foreach my $pluginname (keys %{$PluginsLoaded{'SectionProcessHostname'}}) { my $function="SectionProcessHostname_$pluginname(\$Host)"; eval("$function"); } } } else { if ($PluginsLoaded{'GetCountryCodeByName'}{'geoipfree'}) { $Domain=GetCountryCodeByName_geoipfree($HostResolved); } elsif ($PluginsLoaded{'GetCountryCodeByName'}{'geoip'}) { $Domain=GetCountryCodeByName_geoip($HostResolved); } elsif ($HostResolved =~ /\.(\w+)$/) { $Domain=$1; } if ($AtLeastOneSectionPlugin) { foreach my $pluginname (keys %{$PluginsLoaded{'SectionProcessHostname'}}) { my $function="SectionProcessHostname_$pluginname(\$HostResolved)"; eval("$function"); } } } } # Store country if ($PageBool) { $_domener_p{$Domain}++; } $_domener_h{$Domain}++; $_domener_k{$Domain}+=int($field[$pos_size]); # Analyze: Host, URL entry+exit and Session #------------------------------------------ if ($PageBool) { my $timehostl=$_host_l{$HostResolved}; if ($timehostl) { # A visit for this host was already detected # TODO everywhere there is $VISITTIMEOUT # $timehostl =~ /^\d\d\d\d\d\d(\d\d)/; my $daytimehostl=$1; # if ($timerecord > ($timehostl+$VISITTIMEOUT+($dateparts[3]>$daytimehostl?$NEWDAYVISITTIMEOUT:0))) { if ($timerecord > ($timehostl+$VISITTIMEOUT)) { # This is a second visit or more if (! $_waithost_s{$HostResolved}) { # This is a second visit or more # We count 'visit','exit','entry','DayVisits' if ($Debug) { debug(" This is a second visit for $HostResolved.",4); } my $timehosts=$_host_s{$HostResolved}; my $page=$_host_u{$HostResolved}; if ($page) { $_url_x{$page}++; } $_url_e{$field[$pos_url]}++; $DayVisits{$yearmonthdayrecord}++; # We can't count session yet because we don't have the start so # we save params of first 'wait' session $_waithost_l{$HostResolved}=$timehostl; $_waithost_s{$HostResolved}=$timehosts; $_waithost_u{$HostResolved}=$page; } else { # This is third visit or more # We count 'session','visit','exit','entry','DayVisits' if ($Debug) { debug(" This is a third visit or more for $HostResolved.",4); } my $timehosts=$_host_s{$HostResolved}; my $page=$_host_u{$HostResolved}; if ($page) { $_url_x{$page}++; } $_url_e{$field[$pos_url]}++; $DayVisits{$yearmonthdayrecord}++; if ($timehosts) { $_session{GetSessionRange($timehosts,$timehostl)}++; } } # Save new session properties $_host_s{$HostResolved}=$timerecord; $_host_l{$HostResolved}=$timerecord; $_host_u{$HostResolved}=$field[$pos_url]; } elsif ($timerecord > $timehostl) { # This is a same visit we can count if ($Debug) { debug(" This is same visit still running for $HostResolved. host_l/host_u changed to $timerecord/$field[$pos_url]",4); } $_host_l{$HostResolved}=$timerecord; $_host_u{$HostResolved}=$field[$pos_url]; } elsif ($timerecord == $timehostl) { # This is a same visit we can count if ($Debug) { debug(" This is same visit still running for $HostResolved. host_l/host_u changed to $timerecord/$field[$pos_url]",4); } $_host_u{$HostResolved}=$field[$pos_url]; } elsif ($timerecord < $_host_s{$HostResolved}) { # Should happens only with not correctly sorted log files if ($Debug) { debug(" This is same visit still running for $HostResolved with start not in order. host_s changed to $timerecord (entry page also changed if first visit)",4); } if (! $_waithost_s{$HostResolved}) { # We can reorder entry page only if it's the first visit found in this update run (The saved entry page was $_waithost_e if $_waithost_s{$HostResolved} is not defined. If second visit or more, entry was directly counted and not saved) $_waithost_e{$HostResolved}=$field[$pos_url]; } else { # We can't change entry counted as we dont't know what was the url counted as entry } $_host_s{$HostResolved}=$timerecord; } else { if ($Debug) { debug(" This is same visit still running for $HostResolved with hit between start and last hits. No change",4); } } } else { # This is a new visit (may be). First new visit found for this host. We save in wait array the entry page to count later if ($Debug) { debug(" New session (may be) for $HostResolved. Save in wait array to see later",4); } $_waithost_e{$HostResolved}=$field[$pos_url]; # Save new session properties $_host_u{$HostResolved}=$field[$pos_url]; $_host_s{$HostResolved}=$timerecord; $_host_l{$HostResolved}=$timerecord; } $_host_p{$HostResolved}++; } $_host_h{$HostResolved}++; $_host_k{$HostResolved}+=int($field[$pos_size]); # Analyze: Browser - OS #---------------------- if ($pos_agent >= 0 && $UserAgent) { if ($LevelForBrowsersDetection) { # Analyze: Browser #----------------- my $uabrowser=$TmpBrowser{$UserAgent}; if (! $uabrowser) { my $found=1; # IE ? if ($UserAgent =~ /$regvermsie/o && $UserAgent !~ /$regnotie/o) { $_browser_h{"msie$2"}++; $TmpBrowser{$UserAgent}="msie$2"; } # Netscape 6.x, 7.x ... ? elsif ($UserAgent =~ /$regvernetscape/o) { $_browser_h{"netscape$1"}++; $TmpBrowser{$UserAgent}="netscape$1"; } # Netscape 3.x, 4.x ... ? elsif ($UserAgent =~ /$regvermozilla/o && $UserAgent !~ /$regnotnetscape/o) { $_browser_h{"netscape$2"}++; $TmpBrowser{$UserAgent}="netscape$2"; } # Other known browsers ? else { $found=0; foreach (@BrowsersSearchIDOrder) { # Search ID in order of BrowsersSearchIDOrder if ($UserAgent =~ /$_/) { my $browser=&UnCompileRegex($_); # TODO If browser is in a family, use version $_browser_h{"$browser"}++; $TmpBrowser{$UserAgent}="$browser"; $found=1; last; } } } # Unknown browser ? if (!$found) { $_browser_h{'Unknown'}++; $TmpBrowser{$UserAgent}='Unknown'; my $newua=$UserAgent; $newua =~ tr/\+ /__/; $_unknownrefererbrowser_l{$newua}=$timerecord; } } else { $_browser_h{$uabrowser}++; if ($uabrowser eq 'Unknown') { my $newua=$UserAgent; $newua =~ tr/\+ /__/; $_unknownrefererbrowser_l{$newua}=$timerecord; } } } if ($LevelForOSDetection) { # Analyze: OS #------------ my $uaos=$TmpOS{$UserAgent}; if (! $uaos) { my $found=0; # in OSHashID list ? foreach (@OSSearchIDOrder) { # Search ID in order of OSSearchIDOrder if ($UserAgent =~ /$_/) { my $osid=$OSHashID{&UnCompileRegex($_)}; $_os_h{"$osid"}++; $TmpOS{$UserAgent}="$osid"; $found=1; last; } } # Unknown OS ? if (!$found) { $_os_h{'Unknown'}++; $TmpOS{$UserAgent}='Unknown'; my $newua=$UserAgent; $newua =~ tr/\+ /__/; $_unknownreferer_l{$newua}=$timerecord; } } else { $_os_h{$uaos}++; if ($uaos eq 'Unknown') { my $newua=$UserAgent; $newua =~ tr/\+ /__/; $_unknownreferer_l{$newua}=$timerecord; } } } } else { $_browser_h{'Unknown'}++; $_os_h{'Unknown'}++; } # Analyze: Referer #----------------- my $found=0; if ($pos_referer >= 0 && $LevelForRefererAnalyze && $field[$pos_referer]) { # Direct ? if ($field[$pos_referer] eq '-' || $field[$pos_referer] eq 'bookmarks') { # "bookmarks" is sent by Netscape, '-' by all others browsers # Direct access if ($PageBool) { $_from_p[0]++; } $_from_h[0]++; $found=1; } else { $field[$pos_referer] =~ /$regreferer/o; my $refererprot=$1; my $refererserver=($2||'').(! $3 || $3 eq ':80'?'':$3); # refererserver is www.xxx.com or www.xxx.com:81 but not www.xxx.com:80 # HTML link ? if ($refererprot =~ /^http/i) { #if ($Debug) { debug(" Analyze referer refererprot=$refererprot refererserver=$refererserver",5); } # Kind of origin if (!$TmpRefererServer{$refererserver}) { # is "=" if same site, "search egine key" if search engine, not defined otherwise if ($refererserver =~ /$reglocal/o) { # Intern (This hit came from another page of the site) if ($Debug) { debug(" Server '$refererserver' is added to TmpRefererServer with value '='",2); } $TmpRefererServer{$refererserver}='='; $found=1; } else { foreach (@HostAliases) { if ($refererserver =~ /$_/) { # Intern (This hit came from another page of the site) if ($Debug) { debug(" Server '$refererserver' is added to TmpRefererServer with value '='",2); } $TmpRefererServer{$refererserver}='='; $found=1; last; } } if (! $found) { # Extern (This hit came from an external web site). if ($LevelForSearchEnginesDetection) { foreach (@SearchEnginesSearchIDOrder) { # Search ID in order of SearchEnginesSearchIDOrder if ($refererserver =~ /$_/) { my $key=&UnCompileRegex($_); if (! $NotSearchEnginesKeys{$key} || $refererserver !~ /$NotSearchEnginesKeys{$key}/i) { # This hit came from the search engine $key if ($Debug) { debug(" Server '$refererserver' is added to TmpRefererServer with value '$key'",2); } $TmpRefererServer{$refererserver}=$SearchEnginesHashID{$key}; $found=1; } last; } } } } } } if ($TmpRefererServer{$refererserver}) { if ($TmpRefererServer{$refererserver} eq '=') { # Intern (This hit came from another page of the site) if ($PageBool) { $_from_p[4]++; } $_from_h[4]++; $found=1; } else { # This hit came from a search engine if ($PageBool) { $_from_p[2]++; $_se_referrals_p{$TmpRefererServer{$refererserver}}++; } $_from_h[2]++; $_se_referrals_h{$TmpRefererServer{$refererserver}}++; $found=1; if ($PageBool && $LevelForKeywordsDetection) { # we will complete %_keyphrases hash array my @refurl=split(/\?/,$field[$pos_referer],2); # TODO Use \? or [$URLQuerySeparators] ? if ($refurl[1]) { # Extract params of referer query string (q=cache:mmm:www/zzz+aaa+bbb q=aaa+bbb/ccc key=ddd%20eee lang_en ie=UTF-8 ...) if ($SearchEnginesKnownUrl{$TmpRefererServer{$refererserver}}) { # Search engine with known URL syntax my @paramlist=split(/&/,$KeyWordsNotSensitive?lc($refurl[1]):$refurl[1]); foreach my $param (@paramlist) { if ($param =~ s/^$SearchEnginesKnownUrl{$TmpRefererServer{$refererserver}}//) { # We found good parameter # Now param is keyphrase: "cache:mmm:www/zzz+aaa+bbb/ccc+ddd%20eee'fff,ggg" $param =~ s/^(cache|related):[^\+]+//; # Should be useless since this is for hit on 'not pages' &ChangeWordSeparatorsIntoSpace($param); # Change [ aaa+bbb/ccc+ddd%20eee'fff,ggg ] into [ aaa bbb/ccc ddd eee fff ggg] $param =~ s/^ +//; $param =~ s/ +$//; $param =~ tr/ /\+/s; if ((length $param) > 0) { $_keyphrases{$param}++; } last; } } } elsif ($LevelForKeywordsDetection >= 2) { # Search engine with unknown URL syntax my @paramlist=split(/&/,$KeyWordsNotSensitive?lc($refurl[1]):$refurl[1]); foreach my $param (@paramlist) { my $foundexcludeparam=0; foreach my $paramtoexclude (@WordsToCleanSearchUrl) { if ($param =~ /$paramtoexclude/i) { $foundexcludeparam=1; last; } # Not the param with search criteria } if ($foundexcludeparam) { next; } # We found good parameter $param =~ s/.*=//; # Now param is keyphrase: "aaa+bbb/ccc+ddd%20eee'fff,ggg" $param =~ s/^(cache|related):[^\+]+//; # Should be useless since this is for hit on 'not pages' &ChangeWordSeparatorsIntoSpace($param); # Change [ aaa+bbb/ccc+ddd%20eee'fff,ggg ] into [ aaa bbb/ccc ddd eee fff ggg ] $param =~ s/^ +//; $param =~ s/ +$//; $param =~ tr/ /\+/s; if ((length $param) > 2) { $_keyphrases{$param}++; last; } } } } # End of if refurl[1] } } } # End of if ($TmpRefererServer) else { # This hit came from a site other than a search engine if ($PageBool) { $_from_p[3]++; } $_from_h[3]++; # http://www.mysite.com/ must be same referer than http://www.mysite.com but .../mypage/ differs of .../mypage #if ($refurl[0] =~ /^[^\/]+\/$/) { $field[$pos_referer] =~ s/\/$//; } # Code moved in Save_History # TODO: lowercase the value for referer server to have refering server not case sensitive if ($URLReferrerWithQuery) { if ($PageBool) { $_pagesrefs_p{$field[$pos_referer]}++; } $_pagesrefs_h{$field[$pos_referer]}++; } else { # We discard query for referer if ($field[$pos_referer]=~/$regreferernoquery/o) { if ($PageBool) { $_pagesrefs_p{"$1"}++; } $_pagesrefs_h{"$1"}++; } else { if ($PageBool) { $_pagesrefs_p{$field[$pos_referer]}++; } $_pagesrefs_h{$field[$pos_referer]}++; } } $found=1; } } # News Link ? if (! $found && $refererprot =~ /^news/i) { $found=1; if ($PageBool) { $_from_p[5]++; } $_from_h[5]++; } } } # Origin not found if (!$found) { if ($ShowUnknownOrigin) { print "Unknown origin: $field[$pos_referer]\n"; } if ($PageBool) { $_from_p[1]++; } $_from_h[1]++; } # Analyze: EMail #--------------- if ($pos_emails>=0 && $field[$pos_emails]) { if ($field[$pos_emails] eq '<>') { $field[$pos_emails]='Unknown'; } elsif ($field[$pos_emails] !~ /\@/) { $field[$pos_emails].="\@$SiteDomain"; } $_emails_h{lc($field[$pos_emails])}++; #Count accesses for sender email (hit) $_emails_k{lc($field[$pos_emails])}+=int($field[$pos_size]); #Count accesses for sender email (kb) $_emails_l{lc($field[$pos_emails])}=$timerecord; } if ($pos_emailr>=0 && $field[$pos_emailr]) { if ($field[$pos_emailr] !~ /\@/) { $field[$pos_emailr].="\@$SiteDomain"; } $_emailr_h{lc($field[$pos_emailr])}++; #Count accesses for receiver email (hit) $_emailr_k{lc($field[$pos_emailr])}+=int($field[$pos_size]); #Count accesses for receiver email (kb) $_emailr_l{lc($field[$pos_emailr])}=$timerecord; } } # Check cluster #-------------- if ($pos_cluster>=0) { if ($PageBool) { $_cluster_p{$field[$pos_cluster]}++; } #Count accesses for page (page) $_cluster_h{$field[$pos_cluster]}++; #Count accesses for page (hit) $_cluster_k{$field[$pos_cluster]}+=int($field[$pos_size]); #Count accesses for page (kb) } # Analyze: Extra #--------------- foreach my $extranum (1..@ExtraName-1) { if ($Debug) { debug(" Process extra analyze $extranum",4); } # Check code my $conditionok=0; foreach my $condnum (0..@{$ExtraCodeFilter[$extranum]}-1) { if ($Debug) { debug(" Check code '$field[$pos_code]' must be '$ExtraCodeFilter[$extranum][$condnum]'",5); } if ($field[$pos_code] eq "$ExtraCodeFilter[$extranum][$condnum]") { $conditionok=1; last; } } if (! $conditionok && @{$ExtraCodeFilter[$extranum]}) { next; } # End for this section if ($Debug) { debug(" No check on code or code is OK. Now we check other conditions.",5); } # Check conditions $conditionok=0; foreach my $condnum (0..@{$ExtraConditionType[$extranum]}-1) { my $conditiontype=$ExtraConditionType[$extranum][$condnum]; my $conditiontypeval=$ExtraConditionTypeVal[$extranum][$condnum]; if ($conditiontype eq 'URL') { if ($Debug) { debug(" Check condition '$conditiontype' must contain '$conditiontypeval' in '$urlwithnoquery'",5); } if ($urlwithnoquery =~ /$conditiontypeval/) { $conditionok=1; last; } } elsif ($conditiontype eq 'QUERY_STRING') { if ($Debug) { debug(" Check condition '$conditiontype' must contain '$conditiontypeval' in '$standalonequery'",5); } if ($standalonequery =~ /$conditiontypeval/) { $conditionok=1; last; } } elsif ($conditiontype eq 'REFERER') { if ($Debug) { debug(" Check condition '$conditiontype' must contain '$conditiontypeval' in '$field[$pos_referer]'",5); } if ($field[$pos_referer] =~ /$conditiontypeval/) { $conditionok=1; last; } } elsif ($conditiontype eq 'UA') { if ($Debug) { debug(" Check condition '$conditiontype' must contain '$conditiontypeval' in '$field[$pos_agent]'",5); } if ($field[$pos_agent] =~ /$conditiontypeval/) { $conditionok=1; last; } } elsif ($conditiontype eq 'HOST') { if ($Debug) { debug(" Check condition '$conditiontype' must contain '$conditiontypeval' in '$field[$pos_host]'",5); } if ($HostResolved =~ /$conditiontypeval/) { $conditionok=1; last; } } elsif ($conditiontype =~ /extra(\d+)/i) { if ($Debug) { debug(" Check condition '$conditiontype' must contain '$conditiontypeval' in '$field[$pos_extra[$1]]'",5); } if ($field[$pos_extra[$1]] =~ /$conditiontypeval/) { $conditionok=1; last; } } else { error("Wrong value of parameter ExtraSectionCondition$extranum"); } } if (! $conditionok && @{$ExtraConditionType[$extranum]}) { next; } # End for this section if ($Debug) { debug(" No condition or condition is OK. Now we extract value for first column of extra chart.",5); } # Determine actual column value to use. my $rowkeyval; my $rowkeyok=0; foreach my $rowkeynum (0..@{$ExtraFirstColumnValuesType[$extranum]}-1) { my $rowkeytype=$ExtraFirstColumnValuesType[$extranum][$rowkeynum]; my $rowkeytypeval=$ExtraFirstColumnValuesTypeVal[$extranum][$rowkeynum]; if ($rowkeytype eq 'URL') { if ($urlwithnoquery =~ /$rowkeytypeval/) { $rowkeyval = "$1"; $rowkeyok = 1; last; } } elsif ($rowkeytype eq 'QUERY_STRING') { if ($Debug) { debug(" Extract value from '$standalonequery' with regex '$rowkeytypeval'.",5); } if ($standalonequery =~ /$rowkeytypeval/) { $rowkeyval = "$1"; $rowkeyok = 1; last; } } elsif ($rowkeytype eq 'REFERER') { if ($field[$pos_referer] =~ /$rowkeytypeval/) { $rowkeyval = "$1"; $rowkeyok = 1; last; } } elsif ($rowkeytype eq 'UA') { if ($field[$pos_agent] =~ /$rowkeytypeval/) { $rowkeyval = "$1"; $rowkeyok = 1; last; } } elsif ($rowkeytype eq 'HOST') { if ($HostResolved =~ /$rowkeytypeval/) { $rowkeyval = "$1"; $rowkeyok = 1; last; } } elsif ($rowkeytype =~ /extra(\d+)/i) { if ($field[$pos_extra[$1]] =~ /$rowkeytypeval/) { $rowkeyval = "$1"; $rowkeyok = 1; last; } } else { error("Wrong value of parameter ExtraSectionFirstColumnValues$extranum"); } } if (! $rowkeyok) { next; } # End for this section if ($Debug) { debug(" Key val was found: $rowkeyval",5); } # Here we got all values to increase counters if ($PageBool && $ExtraStatTypes[$extranum] =~ /P/i) { ${'_section_' . $extranum . '_p'}{$rowkeyval}++; } ${'_section_' . $extranum . '_h'}{$rowkeyval}++; # Must be set if ($ExtraStatTypes[$extranum] =~ /B/i) { ${'_section_' . $extranum . '_k'}{$rowkeyval}+=int($field[$pos_size]); } if ($ExtraStatTypes[$extranum] =~ /L/i) { if (${'_section_' . $extranum . '_l'}{$rowkeyval}||0 < $timerecord) { ${'_section_' . $extranum . '_l'}{$rowkeyval}=$timerecord; } } # Check to avoid too large extra sections if (scalar keys %{'_section_' . $extranum . '_h'} > $ExtraTrackedRowsLimit) { error(<= 20000) { #if (++$counterforflushtest >= 1) { if ((scalar keys %_host_u) > ($LIMITFLUSH<<2) || (scalar keys %_url_p) > $LIMITFLUSH) { # warning("Warning: Try to run AWStats update process more frequently to analyze smaler log files."); if ($^X =~ /activestate/i || $^X =~ /activeperl/i) { # We don't flush if perl is activestate to avoid slowing process because of memory hole } else { # Clean tmp hash arrays #%TmpDNSLookup = (); %TmpOS = %TmpRefererServer = %TmpRobot = %TmpBrowser = (); # We flush if perl is not activestate print "Flush history file on disk"; if ((scalar keys %_host_u) > ($LIMITFLUSH<<2)) { print " (unique hosts reach flush limit of ".($LIMITFLUSH<<2).")"; } if ((scalar keys %_url_p) > $LIMITFLUSH) { print " (unique url reach flush limit of ".($LIMITFLUSH).")"; } print "\n"; if ($Debug) { debug("End of set of $counterforflushtest records: Some hash arrays are too large. We flush and clean some.",2); print " _host_p:".(scalar keys %_host_p)." _host_h:".(scalar keys %_host_h)." _host_k:".(scalar keys %_host_k)." _host_l:".(scalar keys %_host_l)." _host_s:".(scalar keys %_host_s)." _host_u:".(scalar keys %_host_u)."\n"; print " _url_p:".(scalar keys %_url_p)." _url_k:".(scalar keys %_url_k)." _url_e:".(scalar keys %_url_e)." _url_x:".(scalar keys %_url_x)."\n"; print " _waithost_e:".(scalar keys %_waithost_e)." _waithost_l:".(scalar keys %_waithost_l)." _waithost_s:".(scalar keys %_waithost_s)." _waithost_u:".(scalar keys %_waithost_u)."\n"; } # &Read_History_With_TmpUpdate($lastprocessedyear,$lastprocessedmonth,1,1,"all",($lastlinenb+$NbOfLinesParsed),$lastlineoffset,&CheckSum($_)); &Read_History_With_TmpUpdate($YearRequired,$MonthRequired,1,1,"all",($lastlinenb+$NbOfLinesParsed),$lastlineoffset,&CheckSum($_)); &GetDelaySinceStart(1); $NbOfLinesShowsteps=1; } } $counterforflushtest=0; } } # End of loop for processing new record. if ($Debug) { debug(" _host_p:".(scalar keys %_host_p)." _host_h:".(scalar keys %_host_h)." _host_k:".(scalar keys %_host_k)." _host_l:".(scalar keys %_host_l)." _host_s:".(scalar keys %_host_s)." _host_u:".(scalar keys %_host_u)."\n",1); debug(" _url_p:".(scalar keys %_url_p)." _url_k:".(scalar keys %_url_k)." _url_e:".(scalar keys %_url_e)." _url_x:".(scalar keys %_url_x)."\n",1); debug(" _waithost_e:".(scalar keys %_waithost_e)." _waithost_l:".(scalar keys %_waithost_l)." _waithost_s:".(scalar keys %_waithost_s)." _waithost_u:".(scalar keys %_waithost_u)."\n",1); debug("End of processing log file (AWStats memory cache is TmpDNSLookup=".(scalar keys %TmpDNSLookup)." TmpBrowser=".(scalar keys %TmpBrowser)." TmpOS=".(scalar keys %TmpOS)." TmpRefererServer=".(scalar keys %TmpRefererServer)." TmpRobot=".(scalar keys %TmpRobot).")",1); } # Save current processed month $lastprocessedmonth # If lastprocessedmonth > 0 means there is at least one approved new record in log or at least one existing history file if ($lastprocessedmonth) { # TODO: Do not save if we are sure a flush was just already done # Get last line seek(LOG,$lastlineoffset,0); my $line=; chomp $line; $line =~ s/\r$//; if (! $NbOfLinesParsed) { # TODO If there was no lines parsed (log was empty), we only update LastUpdate line with YYYYMMDDHHMMSS 0 0 0 0 0 &Read_History_With_TmpUpdate($YearRequired,$MonthRequired,1,1,"all",($lastlinenb+$NbOfLinesParsed),$lastlineoffset,&CheckSum($line)); } else { &Read_History_With_TmpUpdate($YearRequired,$MonthRequired,1,1,"all",($lastlinenb+$NbOfLinesParsed),$lastlineoffset,&CheckSum($line)); } } if ($Debug) { debug("Close log file \"$LogFile\""); } close LOG || error("Command for pipe '$LogFile' failed"); # Process the Rename - Archive - Purge phase my $renameok=1; my $archiveok=1; # Open Log file for writing if PurgeLogFile is on if ($PurgeLogFile == 1) { if ($ArchiveLogRecords == 1) { $ArchiveFileName="$DirData/${PROG}_archive$FileSuffix.log"; open(LOG,"+<$LogFile") || error("Enable to archive log records of \"$LogFile\" into \"$ArchiveFileName\" because source can't be opened for read and write: $!
\n"); } else { open(LOG,"+<$LogFile"); } binmode LOG; } # Rename all HISTORYTMP files into HISTORYTXT &Rename_All_Tmp_History; # Purge Log file if option is on and all renaming are ok if ($PurgeLogFile == 1) { # Archive LOG file into ARCHIVELOG if ($ArchiveLogRecords == 1) { if ($Debug) { debug("Start of archiving log file"); } open(ARCHIVELOG,">>$ArchiveFileName") || error("Couldn't open file \"$ArchiveFileName\" to archive log: $!"); binmode ARCHIVELOG; while () { if (! print ARCHIVELOG $_) { $archiveok=0; last; } } close(ARCHIVELOG) || error("Archiving failed during closing archive: $!"); if ($SaveDatabaseFilesWithPermissionsForEveryone) { chmod 0666,"$ArchiveFileName"; } if ($Debug) { debug("End of archiving log file"); } } # If rename and archive ok if ($renameok && $archiveok) { if ($Debug) { debug("Purge log file"); } my $bold=($ENV{'GATEWAY_INTERFACE'}?'':''); my $unbold=($ENV{'GATEWAY_INTERFACE'}?'':''); my $br=($ENV{'GATEWAY_INTERFACE'}?'
':''); truncate(LOG,0) || warning("Warning: $bold$PROG$unbold couldn't purge logfile \"$bold$LogFile$unbold\".$br\nChange your logfile permissions to allow write for your web server CGI process or change PurgeLogFile=1 into PurgeLogFile=0 in configure file and think to purge sometimes manually your logfile (just after running an update process to not loose any not already processed records your log file contains)."); } close(LOG); } if ($DNSLookup==1 && $DNSLookupAlreadyDone) { # DNSLookup warning my $bold=($ENV{'GATEWAY_INTERFACE'}?'':''); my $unbold=($ENV{'GATEWAY_INTERFACE'}?'':''); my $br=($ENV{'GATEWAY_INTERFACE'}?'
':''); warning("Warning: $bold$PROG$unbold has detected that some hosts names were already resolved in your logfile $bold$DNSLookupAlreadyDone$unbold.$br\nIf DNS lookup was already made by the logger (web server), you should change your setup DNSLookup=$DNSLookup into DNSLookup=0 to increase $PROG speed."); } if ($DNSLookup==1 && $NbOfNewLines) { # Save new DNS last update cache file Save_DNS_Cache_File(\%TmpDNSLookup,"$DirData/$DNSLastUpdateCacheFile","$FileSuffix"); # Save into file using FileSuffix } if ($EnableLockForUpdate) { # Remove lock &Lock_Update(0); # Restore signals handler $SIG{INT} = 'DEFAULT'; # 2 #$SIG{KILL} = 'DEFAULT'; # 9 #$SIG{TERM} = 'DEFAULT'; # 15 } } # close Pablo2; #--------------------------------------------------------------------- # SHOW REPORT #--------------------------------------------------------------------- if (scalar keys %HTMLOutput) { my $max_p; my $max_h; my $max_k; my $max_v; my $total_u; my $total_v; my $total_p; my $total_h; my $total_k; my $total_e; my $total_x; my $total_s; my $total_l; my $total_r; my $average_u; my $average_v; my $average_p; my $average_h; my $average_k; my $average_s; my $rest_p; my $rest_h; my $rest_k; my $rest_e; my $rest_x; my $rest_s; my $rest_l; my $rest_r; my $average_nb; # Define the NewLinkParams for main chart my $NewLinkParams=${QueryString}; $NewLinkParams =~ s/(^|&)update(=\w*|$)//i; $NewLinkParams =~ s/(^|&)output(=\w*|$)//i; $NewLinkParams =~ s/(^|&)staticlinks(=\w*|$)//i; $NewLinkParams =~ s/(^|&)framename=[^&]*//i; my $NewLinkTarget=''; if ($DetailedReportsOnNewWindows) { $NewLinkTarget=" target=\"awstatsbis\""; } if (($FrameName eq 'mainleft' || $FrameName eq 'mainright') && $DetailedReportsOnNewWindows < 2) { $NewLinkParams.="&framename=mainright"; $NewLinkTarget=" target=\"mainright\""; } $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; if ($NewLinkParams) { $NewLinkParams="${NewLinkParams}&"; } if ($FrameName ne 'mainleft') { # READING DATA #------------- &Init_HashArray(); # Loop on each month of year for (my $ix=12; $ix>=1; $ix--) { my $monthix=sprintf("%02s",$ix); if ($MonthRequired eq 'all' || $monthix eq $MonthRequired) { &Read_History_With_TmpUpdate($YearRequired,$monthix,0,0,"all"); # Read full history file } elsif (($HTMLOutput{'main'} && $ShowMonthStats) || $HTMLOutput{'alldays'}) { &Read_History_With_TmpUpdate($YearRequired,$monthix,0,0,"general time"); # Read general and time sections. } } } # HTMLHeadSection if ($FrameName ne 'index' && $FrameName ne 'mainleft') { print " \n\n"; print "$HTMLHeadSection\n"; print "\n"; } # Call to plugins' function AddHTMLBodyHeader foreach my $pluginname (keys %{$PluginsLoaded{'AddHTMLBodyHeader'}}) { my $function="AddHTMLBodyHeader_$pluginname()"; eval("$function"); } my $WIDTHMENU1=($FrameName eq 'mainleft'?$FRAMEWIDTH:150); # TOP BAN #--------------------------------------------------------------------- if ($ShowMenu || $FrameName eq 'mainleft') { my $frame=($FrameName eq 'mainleft'); if ($Debug) { debug("ShowTopBan",2); } print "$Center \n"; if ($FrameName ne 'mainleft') { my $NewLinkParams=${QueryString}; $NewLinkParams =~ s/(^|&)update(=\w*|$)//i; $NewLinkParams =~ s/(^|&)staticlinks(=\w*|$)//i; $NewLinkParams =~ s/(^|&)year=[^&]*//i; $NewLinkParams =~ s/(^|&)month=[^&]*//i; $NewLinkParams =~ s/(^|&)framename=[^&]*//i; $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; my $NewLinkTarget=''; if ($FrameName eq 'mainright') { $NewLinkTarget=" target=\"_parent\""; } print "
\n"; } if ($QueryString !~ /buildpdf/i) { print "\n"; print "
\n"; print "\n"; } else { print "
\n"; } if ($FrameName ne 'mainright') { # Print Statistics Of if ($FrameName eq 'mainleft') { print ""; } else { print ""; } # Logo and flags if ($FrameName ne 'mainleft') { if ($LogoLink =~ "http://awstats.sourceforge.net") { print ""; } print "\n"; } if ($FrameName ne 'mainleft') { # Print Last Update print ""; print ""; # Logo and flags if ($FrameName eq 'mainright') { if ($LogoLink =~ "http://awstats.sourceforge.net") { print ""; } print "\n"; # Print selected period of analysis (month and year required) print ""; print "\n"; } if ($QueryString !~ /buildpdf/i) { print "
$Message[7]:
$SiteDomain
$Message[7]: $SiteDomain"; } else { print ""; } if (! $StaticLinks) { print "
"; Show_Flag_Links($Lang); } print "
$Message[35]: "; if ($LastUpdate) { print Format_Date($LastUpdate,0); } else { # Here NbOfOldLines = 0 (because LastUpdate is not defined) if (! $UpdateStats) { print "$Message[24]"; } else { print "No qualified records found in log ($NbOfLinesCorrupted corrupted, $NbOfLinesDropped dropped)"; } } print ""; # Print Update Now link if ($AllowToUpdateStatsFromBrowser && ! $StaticLinks) { my $NewLinkParams=${QueryString}; $NewLinkParams =~ s/(^|&)update(=\w*|$)//i; $NewLinkParams =~ s/(^|&)staticlinks(=\w*|$)//i; $NewLinkParams =~ s/(^|&)framename=[^&]*//i; if ($FrameName eq 'mainright') { $NewLinkParams.="&framename=mainright"; } $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; if ($NewLinkParams) { $NewLinkParams="${NewLinkParams}&"; } print "       "; print "$Message[74]"; } print "\n"; } else { print "\n"; } if (! $StaticLinks) { print "
"; Show_Flag_Links($Lang); } print "
$Message[133]:"; if ($ENV{'GATEWAY_INTERFACE'} || !$StaticLinks) { print "\n"; print "\n"; print "\n"; if ($SiteConfig) { print "\n"; } if ($DirConfig) { print "\n"; } if ($QueryString =~ /onlyusers=([^&]+)/i) { my $temp_3=$1; $temp_3=~ s/\Q%20\E/ /g; $temp_3=~ s/\+/ /g; $temp_3=~ s/\Q%2B\E/ /g; print "\n"; } if ($QueryString =~ /onlylines=([^&]+)/i) { my $temp_4=$1; $temp_4=~ s/\Q%20\E/ /g; $temp_4=~ s/\+/ /g; $temp_4=~ s/\Q%2B\E/ /g; print "\n"; } if ($SiteDomain) { print "\n"; } if ($QueryString =~ /lang=(\w+)/i) { print "\n"; } if ($QueryString =~ /debug=(\d+)/i) { print "\n"; } if ($FrameName eq 'mainright') { print "\n"; } print ""; } else { print ""; if ($MonthRequired eq 'all') { print "$Message[6] $YearRequired"; } else { print "$Message[5] $MonthNumLib{$MonthRequired} $YearRequired"; } print ""; } print "
\n"; print "
\n"; } else { print "\n"; } if ($FrameName ne 'mainleft') { print "
\n"; } else { print "
\n"; } print "\n"; } # Call to plugins' function AddHTMLMenuHeader foreach my $pluginname (keys %{$PluginsLoaded{'AddHTMLMenuHeader'}}) { my $function="AddHTMLMenuHeader_$pluginname()"; eval("$function"); } # MENU #--------------------------------------------------------------------- if ($ShowMenu || $FrameName eq 'mainleft') { my $frame=($FrameName eq 'mainleft'); if ($Debug) { debug("ShowMenu",2); } # Print menu links if (($HTMLOutput{'main'} && $FrameName ne 'mainright') || $FrameName eq 'mainleft') { # If main page asked # Define link anchor my $linkanchor=($FrameName eq 'mainleft'?"$AWScript?${NewLinkParams}":""); if ($linkanchor && ($linkanchor !~ /framename=mainright/)) { $linkanchor.="framename=mainright"; } $linkanchor =~ s/&$//; $linkanchor=XMLEncode("$linkanchor"); # Define target my $targetpage=($FrameName eq 'mainleft'?" target=\"mainright\"":""); # Print Menu my $linetitle; # TODO a virer if (! $PluginsLoaded{'ShowMenu'}{'menuapplet'}) { my $menuicon=0; # TODO a virer # Menu HTML print "\n"; if ($FrameName eq 'mainleft' && $ShowMonthStats) { print ($frame?"":""); print "$Message[128]"; print ($frame?"\n":"   "); } my %menu=(); my %menulink=(); my %menutext=(); # When %menu=('month'=>$ShowMonthStats?1:0,'daysofmonth'=>$ShowDaysOfMonthStats?2:0,'daysofweek'=>$ShowDaysOfWeekStats?3:0,'hours'=>$ShowHoursStats?4:0); %menulink=('month'=>1,'daysofmonth'=>1,'daysofweek'=>1,'hours'=>1); %menutext=('month'=>$Message[162],'daysofmonth'=>$Message[138],'daysofweek'=>$Message[91],'hours'=>$Message[20]); ShowMenuCateg('when',$Message[93],'menu4.png',$frame,$targetpage,$linkanchor,$NewLinkParams,$NewLinkTarget,\%menu,\%menulink,\%menutext); # Who %menu=('countries'=>$ShowDomainsStats?1:0,'alldomains'=>$ShowDomainsStats?2:0,'visitors'=>$ShowHostsStats?3:0,'allhosts'=>$ShowHostsStats?4:0,'lasthosts'=>($ShowHostsStats =~ /L/i)?5:0,'unknownip'=>$ShowHostsStats?6:0,'logins'=>$ShowAuthenticatedUsers?7:0,'alllogins'=>$ShowAuthenticatedUsers?8:0,'lastlogins'=>($ShowAuthenticatedUsers =~ /L/i)?9:0,'emailsenders'=>$ShowEMailSenders?10:0,'allemails'=>$ShowEMailSenders?11:0,'lastemails'=>($ShowEMailSenders =~ /L/i)?12:0,'emailreceivers'=>$ShowEMailReceivers?13:0,'allemailr'=>$ShowEMailReceivers?14:0,'lastemailr'=>($ShowEMailReceivers =~ /L/i)?15:0,'robots'=>$ShowRobotsStats?16:0,'allrobots'=>$ShowRobotsStats?17:0,'lastrobots'=>($ShowRobotsStats =~ /L/i)?18:0,'worms'=>$ShowWormsStats?19:0); %menulink=('countries'=>1,'alldomains'=>2,'visitors'=>1,'allhosts'=>2,'lasthosts'=>2,'unknownip'=>2,'logins'=>1,'alllogins'=>2,'lastlogins'=>2,'emailsenders'=>1,'allemails'=>2,'lastemails'=>2,'emailreceivers'=>1,'allemailr'=>2,'lastemailr'=>2,'robots'=>1,'allrobots'=>2,'lastrobots'=>2,'worms'=>1); %menutext=('countries'=>$Message[148],'alldomains'=>$Message[80],'visitors'=>$Message[81],'allhosts'=>$Message[80],'lasthosts'=>$Message[9],'unknownip'=>$Message[45],'logins'=>$Message[94],'alllogins'=>$Message[80],'lastlogins'=>$Message[9],'emailsenders'=>$Message[131],'allemails'=>$Message[80],'lastemails'=>$Message[9],'emailreceivers'=>$Message[132],'allemailr'=>$Message[80],'lastemailr'=>$Message[9],'robots'=>$Message[53],'allrobots'=>$Message[80],'lastrobots'=>$Message[9],'worms'=>$Message[136]); ShowMenuCateg('who',$Message[92],'menu5.png',$frame,$targetpage,$linkanchor,$NewLinkParams,$NewLinkTarget,\%menu,\%menulink,\%menutext); # Navigation $linetitle=&AtLeastOneNotNull($ShowSessionsStats,$ShowPagesStats,$ShowFileTypesStats,$ShowFileSizesStats,$ShowOSStats,$ShowBrowsersStats,$ShowScreenSizeStats); if ($linetitle) { print "".($menuicon?" ":"")."$Message[72]:\n"; } if ($linetitle) { print ($frame?"\n":""); } if ($ShowSessionsStats) { print ($frame?"":""); print "$Message[117]"; print ($frame?"\n":"   "); } if ($ShowFileTypesStats) { print ($frame?"":""); print "$Message[73]"; print ($frame?"\n":"   "); } if ($ShowPagesStats) { print ($frame?"":""); print "$Message[29]\n"; print ($frame?"\n":"   "); } if ($ShowPagesStats) { print ($frame?"   \"...\" ":""); print "$Message[80]\n"; print ($frame?"\n":"   "); } if ($ShowPagesStats =~ /E/i) { print ($frame?"   \"...\" ":""); print "$Message[104]\n"; print ($frame?"\n":"   "); } if ($ShowPagesStats =~ /X/i) { print ($frame?"   \"...\" ":""); print "$Message[116]\n"; print ($frame?"\n":"   "); } if ($ShowOSStats) { print ($frame?"":""); print "$Message[59]"; print ($frame?"\n":"   "); } if ($ShowOSStats) { print ($frame?"   \"...\" ":""); print "$Message[58]\n"; print ($frame?"\n":"   "); } if ($ShowOSStats) { print ($frame?"   \"...\" ":""); print "$Message[0]\n"; print ($frame?"\n":"   "); } if ($ShowBrowsersStats) { print ($frame?"":""); print "$Message[21]"; print ($frame?"\n":"   "); } if ($ShowBrowsersStats) { print ($frame?"   \"...\" ":""); print "$Message[58]\n"; print ($frame?"\n":"   "); } if ($ShowBrowsersStats) { print ($frame?"   \"...\" ":""); print "$Message[0]\n"; print ($frame?"\n":"   "); } if ($ShowScreenSizeStats) { print ($frame?"":""); print "$Message[135]"; print ($frame?"\n":"   "); } if ($linetitle) { print ($frame?"":"\n"); } # Referers %menu=('referer'=>$ShowOriginStats?1:0,'refererse'=>$ShowOriginStats?2:0,'refererpages'=>$ShowOriginStats?3:0,'keys'=>($ShowKeyphrasesStats || $ShowKeywordsStats)?4:0,'keyphrases'=>$ShowKeyphrasesStats?5:0,'keywords'=>$ShowKeywordsStats?6:0); %menulink=('referer'=>1,'refererse'=>2,'refererpages'=>2,'keys'=>1,'keyphrases'=>2,'keywords'=>2); %menutext=('referer'=>$Message[37],'refererse'=>$Message[126],'refererpages'=>$Message[127],'keys'=>$Message[14],'keyphrases'=>$Message[120],'keywords'=>$Message[121]); ShowMenuCateg('referers',$Message[23],'menu7.png',$frame,$targetpage,$linkanchor,$NewLinkParams,$NewLinkTarget,\%menu,\%menulink,\%menutext); # Others %menu=('filetypes'=>($ShowFileTypesStats =~ /C/i)?1:0,'misc'=>$ShowMiscStats?2:0,'errors'=>($ShowHTTPErrorsStats||$ShowSMTPErrorsStats)?3:0,'clusters'=>$ShowClusterStats?5:0); %menulink=('filetypes'=>1,'misc'=>1,'errors'=>1,'clusters'=>1); %menutext=('filetypes'=>$Message[98],'misc'=>$Message[139],'errors'=>($ShowSMTPErrorsStats?$Message[147]:$Message[32]),'clusters'=>$Message[155]); foreach (keys %TrapInfosForHTTPErrorCodes) { $menu{"errors$_"}=$ShowHTTPErrorsStats?4:0; $menulink{"errors$_"}=2; $menutext{"errors$_"}=$Message[31]; } ShowMenuCateg('others',$Message[2],'menu8.png',$frame,$targetpage,$linkanchor,$NewLinkParams,$NewLinkTarget,\%menu,\%menulink,\%menutext); # Extra/Marketing %menu=(); %menulink=(); %menutext=(); foreach (1..@ExtraName-1) { $menu{"extra$_"}=1; $menulink{"extra$_"}=1; $menutext{"extra$_"}=$ExtraName[$_]; } ShowMenuCateg('extra',$Message[134],'',$frame,$targetpage,$linkanchor,$NewLinkParams,$NewLinkTarget,\%menu,\%menulink,\%menutext); print "\n"; } else { # Menu Applet if ($frame) { } else {} } #print ($frame?"":"
\n"); print "
\n"; } # Print Back link elsif (! $HTMLOutput{'main'}) { print "\n"; $NewLinkParams =~ s/(^|&)hostfilter=[^&]*//i; $NewLinkParams =~ s/(^|&)urlfilter=[^&]*//i; $NewLinkParams =~ s/(^|&)refererpagesfilter=[^&]*//i; $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/&$//; if (! $DetailedReportsOnNewWindows || $FrameName eq 'mainright' || $QueryString =~ /buildpdf/i) { print "\n"; } else { print "\n"; } print "
$Message[76]
$Message[118]
\n"; print "\n"; } } # Call to plugins' function AddHTMLMenuFooter foreach my $pluginname (keys %{$PluginsLoaded{'AddHTMLMenuFooter'}}) { my $function="AddHTMLMenuFooter_$pluginname()"; eval("$function"); } # Exit if left frame if ($FrameName eq 'mainleft') { &html_end(0); exit 0; } # FirstTime LastTime TotalVisits TotalUnique TotalPages TotalHits TotalBytes TotalHostsKnown TotalHostsUnknown my $FirstTime=0; my $LastTime=0; $TotalUnique=$TotalVisits=$TotalPages=$TotalHits=$TotalBytes=0; $TotalNotViewedPages=$TotalNotViewedHits=$TotalNotViewedBytes=0; $TotalHostsKnown=$TotalHostsUnknown=0; my $beginmonth=$MonthRequired;my $endmonth=$MonthRequired; if ($MonthRequired eq 'all') { $beginmonth=1;$endmonth=12; } for (my $month=$beginmonth; $month<=$endmonth; $month++) { my $monthix=sprintf("%02s",$month); if ($FirstTime{$YearRequired.$monthix} && ($FirstTime == 0 || $FirstTime > $FirstTime{$YearRequired.$monthix})) { $FirstTime = $FirstTime{$YearRequired.$monthix}; } if ($LastTime < ($LastTime{$YearRequired.$monthix}||0)) { $LastTime = $LastTime{$YearRequired.$monthix}; } $TotalHostsKnown+=$MonthHostsKnown{$YearRequired.$monthix}||0; # Wrong in year view $TotalHostsUnknown+=$MonthHostsUnknown{$YearRequired.$monthix}||0; # Wrong in year view $TotalUnique+=$MonthUnique{$YearRequired.$monthix}||0; # Wrong in year view $TotalVisits+=$MonthVisits{$YearRequired.$monthix}||0; $TotalPages+=$MonthPages{$YearRequired.$monthix}||0; $TotalHits+=$MonthHits{$YearRequired.$monthix}||0; $TotalBytes+=$MonthBytes{$YearRequired.$monthix}||0; $TotalNotViewedPages+=$MonthNotViewedPages{$YearRequired.$monthix}||0; $TotalNotViewedHits+=$MonthNotViewedHits{$YearRequired.$monthix}||0; $TotalNotViewedBytes+=$MonthNotViewedBytes{$YearRequired.$monthix}||0; } # TotalHitsErrors TotalBytesErrors my $TotalHitsErrors=0; my $TotalBytesErrors=0; foreach (keys %_errors_h) { $TotalHitsErrors+=$_errors_h{$_}; $TotalBytesErrors+=$_errors_k{$_}; } # TotalEntries (if not already specifically counted, we init it from _url_e hash table) if (!$TotalEntries) { foreach (keys %_url_e) { $TotalEntries+=$_url_e{$_}; } } # TotalExits (if not already specifically counted, we init it from _url_x hash table) if (!$TotalExits) { foreach (keys %_url_x) { $TotalExits+=$_url_x{$_}; } } # TotalBytesPages (if not already specifically counted, we init it from _url_k hash table) if (!$TotalBytesPages) { foreach (keys %_url_k) { $TotalBytesPages+=$_url_k{$_}; } } # TotalKeyphrases (if not already specifically counted, we init it from _keyphrases hash table) if (!$TotalKeyphrases) { foreach (keys %_keyphrases) { $TotalKeyphrases+=$_keyphrases{$_}; } } # TotalKeywords (if not already specifically counted, we init it from _keywords hash table) if (!$TotalKeywords) { foreach (keys %_keywords) { $TotalKeywords+=$_keywords{$_}; } } # TotalSearchEnginesPages (if not already specifically counted, we init it from _se_referrals_p hash table) if (!$TotalSearchEnginesPages) { foreach (keys %_se_referrals_p) { $TotalSearchEnginesPages+=$_se_referrals_p{$_}; } } # TotalSearchEnginesHits (if not already specifically counted, we init it from _se_referrals_h hash table) if (!$TotalSearchEnginesHits) { foreach (keys %_se_referrals_h) { $TotalSearchEnginesHits+=$_se_referrals_h{$_}; } } # TotalRefererPages (if not already specifically counted, we init it from _pagesrefs_p hash table) if (!$TotalRefererPages) { foreach (keys %_pagesrefs_p) { $TotalRefererPages+=$_pagesrefs_p{$_}; } } # TotalRefererHits (if not already specifically counted, we init it from _pagesrefs_h hash table) if (!$TotalRefererHits) { foreach (keys %_pagesrefs_h) { $TotalRefererHits+=$_pagesrefs_h{$_}; } } # TotalDifferentPages (if not already specifically counted, we init it from _url_p hash table) $TotalDifferentPages||=scalar keys %_url_p; # TotalDifferentKeyphrases (if not already specifically counted, we init it from _keyphrases hash table) $TotalDifferentKeyphrases||=scalar keys %_keyphrases; # TotalDifferentKeywords (if not already specifically counted, we init it from _keywords hash table) $TotalDifferentKeywords||=scalar keys %_keywords; # TotalDifferentSearchEngines (if not already specifically counted, we init it from _se_referrals_h hash table) $TotalDifferentSearchEngines||=scalar keys %_se_referrals_h; # TotalDifferentReferer (if not already specifically counted, we init it from _pagesrefs_h hash table) $TotalDifferentReferer||=scalar keys %_pagesrefs_h; # Define firstdaytocountaverage, lastdaytocountaverage, firstdaytoshowtime, lastdaytoshowtime my $firstdaytocountaverage=$nowyear.$nowmonth."01"; # Set day cursor to 1st day of month my $firstdaytoshowtime=$nowyear.$nowmonth."01"; # Set day cursor to 1st day of month my $lastdaytocountaverage=$nowyear.$nowmonth.$nowday; # Set day cursor to today my $lastdaytoshowtime=$nowyear.$nowmonth."31"; # Set day cursor to last day of month if ($MonthRequired eq 'all') { $firstdaytocountaverage=$YearRequired."0101"; # Set day cursor to 1st day of the required year } if (($MonthRequired ne $nowmonth && $MonthRequired ne 'all') || $YearRequired ne $nowyear) { if ($MonthRequired eq 'all') { $firstdaytocountaverage=$YearRequired."0101"; # Set day cursor to 1st day of the required year $firstdaytoshowtime=$YearRequired."1201"; # Set day cursor to 1st day of last month of required year $lastdaytocountaverage=$YearRequired."1231"; # Set day cursor to last day of the required year $lastdaytoshowtime=$YearRequired."1231"; # Set day cursor to last day of last month of required year } else { $firstdaytocountaverage=$YearRequired.$MonthRequired."01"; # Set day cursor to 1st day of the required month $firstdaytoshowtime=$YearRequired.$MonthRequired."01"; # Set day cursor to 1st day of the required month $lastdaytocountaverage=$YearRequired.$MonthRequired."31"; # Set day cursor to last day of the required month $lastdaytoshowtime=$YearRequired.$MonthRequired."31"; # Set day cursor to last day of the required month } } if ($Debug) { debug("firstdaytocountaverage=$firstdaytocountaverage, lastdaytocountaverage=$lastdaytocountaverage",1); debug("firstdaytoshowtime=$firstdaytoshowtime, lastdaytoshowtime=$lastdaytoshowtime",1); } # Call to plugins' function AddHTMLContentHeader foreach my $pluginname (keys %{$PluginsLoaded{'AddHTMLContentHeader'}}) { my $function="AddHTMLContentHeader_$pluginname()"; eval("$function"); } # Output particular part #----------------------- if (scalar keys %HTMLOutput == 1) { if ($HTMLOutput{'alldomains'}) { print "$Center 
\n"; # Show domains list my $title=''; my $cpt=0; if ($HTMLOutput{'alldomains'}) { $title.="$Message[25]"; $cpt=(scalar keys %_domener_h); } &tab_head("$title",19,0,'domains'); print " $Message[17]"; if ($ShowDomainsStats =~ /P/i) { print "$Message[56]"; } if ($ShowDomainsStats =~ /H/i) { print "$Message[57]"; } if ($ShowDomainsStats =~ /B/i) { print "$Message[75]"; } print " "; print "\n"; $total_p=$total_h=$total_k=0; $max_h=1; foreach (values %_domener_h) { if ($_ > $max_h) { $max_h = $_; } } $max_k=1; foreach (values %_domener_k) { if ($_ > $max_k) { $max_k = $_; } } my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,1,\%_domener_h,\%_domener_p); foreach my $key (@keylist) { my $bredde_p=0;my $bredde_h=0;my $bredde_k=0; if ($max_h > 0) { $bredde_p=int($BarWidth*$_domener_p{$key}/$max_h)+1; } # use max_h to enable to compare pages with hits if ($_domener_p{$key} && $bredde_p==1) { $bredde_p=2; } if ($max_h > 0) { $bredde_h=int($BarWidth*$_domener_h{$key}/$max_h)+1; } if ($_domener_h{$key} && $bredde_h==1) { $bredde_h=2; } if ($max_k > 0) { $bredde_k=int($BarWidth*($_domener_k{$key}||0)/$max_k)+1; } if ($_domener_k{$key} && $bredde_k==1) { $bredde_k=2; } my $newkey=lc($key); if ($newkey eq 'ip' || ! $DomainsHashIDLib{$newkey}) { print "$Message[0]$newkey"; } else { print "$DomainsHashIDLib{$newkey}$newkey"; } if ($ShowDomainsStats =~ /P/i) { print "$_domener_p{$key}"; } if ($ShowDomainsStats =~ /H/i) { print "$_domener_h{$key}"; } if ($ShowDomainsStats =~ /B/i) { print "".Format_Bytes($_domener_k{$key}).""; } print ""; if ($ShowDomainsStats =~ /P/i) { print "
\n"; } if ($ShowDomainsStats =~ /H/i) { print "
\n"; } if ($ShowDomainsStats =~ /B/i) { print ""; } print ""; print "\n"; $total_p += $_domener_p{$key}; $total_h += $_domener_h{$key}; $total_k += $_domener_k{$key}||0; $count++; } $rest_p=$TotalPages-$total_p; $rest_h=$TotalHits-$total_h; $rest_k=$TotalBytes-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other domains (known or not) print " $Message[2]"; if ($ShowDomainsStats =~ /P/i) { print "$rest_p"; } if ($ShowDomainsStats =~ /H/i) { print "$rest_h"; } if ($ShowDomainsStats =~ /B/i) { print "".Format_Bytes($rest_k).""; } print " "; print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'allhosts'} || $HTMLOutput{'lasthosts'}) { print "$Center 
\n"; # Show filter form &ShowFormFilter("hostfilter",$FilterIn{'host'},$FilterEx{'host'}); # Show hosts list my $title=''; my $cpt=0; if ($HTMLOutput{'allhosts'}) { $title.="$Message[81]"; $cpt=(scalar keys %_host_h); } if ($HTMLOutput{'lasthosts'}) { $title.="$Message[9]"; $cpt=(scalar keys %_host_h); } &tab_head("$title",19,0,'hosts'); print ""; if ($FilterIn{'host'} || $FilterEx{'host'}) { # With filter if ($FilterIn{'host'}) { print "$Message[79] '$FilterIn{'host'}'"; } if ($FilterIn{'host'} && $FilterEx{'host'}) { print " - "; } if ($FilterEx{'host'}) { print " Exlude $Message[79] '$FilterEx{'host'}'"; } if ($FilterIn{'host'} || $FilterEx{'host'}) { print ": "; } print "$cpt $Message[81]"; if ($MonthRequired ne 'all') { if ($HTMLOutput{'allhosts'} || $HTMLOutput{'lasthosts'}) { print "
$Message[102]: $TotalHostsKnown $Message[82], $TotalHostsUnknown $Message[1] - $TotalUnique $Message[11]"; } } } else { # Without filter if ($MonthRequired ne 'all') { print "$Message[102] : $TotalHostsKnown $Message[82], $TotalHostsUnknown $Message[1] - $TotalUnique $Message[11]"; } else { print "$Message[102] : ".(scalar keys %_host_h); } } print ""; &ShowHostInfo('__title__'); if ($ShowHostsStats =~ /P/i) { print "$Message[56]"; } if ($ShowHostsStats =~ /H/i) { print "$Message[57]"; } if ($ShowHostsStats =~ /B/i) { print "$Message[75]"; } if ($ShowHostsStats =~ /L/i) { print "$Message[9]"; } print "\n"; $total_p=$total_h=$total_k=0; my $count=0; if ($HTMLOutput{'allhosts'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Host'},\%_host_h,\%_host_p); } if ($HTMLOutput{'lasthosts'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Host'},\%_host_h,\%_host_l); } foreach my $key (@keylist) { my $host=CleanFromCSSA($key); print "".($_robot_l{$key}?'':'')."$host".($_robot_l{$key}?'':'').""; &ShowHostInfo($key); if ($ShowHostsStats =~ /P/i) { print "".($_host_p{$key}?$_host_p{$key}:" ").""; } if ($ShowHostsStats =~ /H/i) { print "$_host_h{$key}"; } if ($ShowHostsStats =~ /B/i) { print "".Format_Bytes($_host_k{$key}).""; } if ($ShowHostsStats =~ /L/i) { print "".($_host_l{$key}?Format_Date($_host_l{$key},1):'-').""; } print "\n"; $total_p += $_host_p{$key}; $total_h += $_host_h{$key}; $total_k += $_host_k{$key}||0; $count++; } if ($Debug) { debug("Total real / shown : $TotalPages / $total_p - $TotalHits / $total_h - $TotalBytes / $total_h",2); } $rest_p=$TotalPages-$total_p; $rest_h=$TotalHits-$total_h; $rest_k=$TotalBytes-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other visitors (known or not) print "$Message[2]"; &ShowHostInfo(''); if ($ShowHostsStats =~ /P/i) { print "".($rest_p?$rest_p:" ").""; } if ($ShowHostsStats =~ /H/i) { print "$rest_h"; } if ($ShowHostsStats =~ /B/i) { print "".Format_Bytes($rest_k).""; } if ($ShowHostsStats =~ /L/i) { print " "; } print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'unknownip'}) { print "$Center 
\n"; &tab_head("$Message[45]",19,0,'unknownwip'); print "".(scalar keys %_host_h)." $Message[1]"; &ShowHostInfo('__title__'); if ($ShowHostsStats =~ /P/i) { print "$Message[56]"; } if ($ShowHostsStats =~ /H/i) { print "$Message[57]"; } if ($ShowHostsStats =~ /B/i) { print "$Message[75]"; } if ($ShowHostsStats =~ /L/i) { print "$Message[9]"; } print "\n"; $total_p=$total_h=$total_k=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Host'},\%_host_h,\%_host_p); foreach my $key (@keylist) { my $host=CleanFromCSSA($key); print "$host"; &ShowHostInfo($key); if ($ShowHostsStats =~ /P/i) { print "".($_host_p{$key}?$_host_p{$key}:" ").""; } if ($ShowHostsStats =~ /H/i) { print "$_host_h{$key}"; } if ($ShowHostsStats =~ /B/i) { print "".Format_Bytes($_host_k{$key}).""; } if ($ShowHostsStats =~ /L/i) { print "".($_host_l{$key}?Format_Date($_host_l{$key},1):'-').""; } print "\n"; $total_p += $_host_p{$key}; $total_h += $_host_h{$key}; $total_k += $_host_k{$key}||0; $count++; } if ($Debug) { debug("Total real / shown : $TotalPages / $total_p - $TotalHits / $total_h - $TotalBytes / $total_h",2); } $rest_p=$TotalPages-$total_p; $rest_h=$TotalHits-$total_h; $rest_k=$TotalBytes-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other visitors (known or not) print "$Message[82]"; &ShowHostInfo(''); if ($ShowHostsStats =~ /P/i) { print "".($rest_p?$rest_p:" ").""; } if ($ShowHostsStats =~ /H/i) { print "$rest_h"; } if ($ShowHostsStats =~ /B/i) { print "".Format_Bytes($rest_k).""; } if ($ShowHostsStats =~ /L/i) { print " "; } print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'allemails'} || $HTMLOutput{'lastemails'}) { &ShowEmailSendersChart($NewLinkParams,$NewLinkTarget); &html_end(1); } if ($HTMLOutput{'allemailr'} || $HTMLOutput{'lastemailr'}) { &ShowEmailReceiversChart($NewLinkParams,$NewLinkTarget); &html_end(1); } if ($HTMLOutput{'alllogins'} || $HTMLOutput{'lastlogins'}) { print "$Center 
\n"; my $title=''; if ($HTMLOutput{'alllogins'}) { $title.="$Message[94]"; } if ($HTMLOutput{'lastlogins'}) { $title.="$Message[9]"; } &tab_head("$title",19,0,'logins'); print "$Message[94] : ".(scalar keys %_login_h).""; &ShowUserInfo('__title__'); if ($ShowAuthenticatedUsers =~ /P/i) { print "$Message[56]"; } if ($ShowAuthenticatedUsers =~ /H/i) { print "$Message[57]"; } if ($ShowAuthenticatedUsers =~ /B/i) { print "$Message[75]"; } if ($ShowAuthenticatedUsers =~ /L/i) { print "$Message[9]"; } print "\n"; $total_p=$total_h=$total_k=0; my $count=0; if ($HTMLOutput{'alllogins'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Host'},\%_login_h,\%_login_p); } if ($HTMLOutput{'lastlogins'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Host'},\%_login_h,\%_login_l); } foreach my $key (@keylist) { print "$key"; &ShowUserInfo($key); if ($ShowAuthenticatedUsers =~ /P/i) { print "".($_login_p{$key}?$_login_p{$key}:" ").""; } if ($ShowAuthenticatedUsers =~ /H/i) { print "$_login_h{$key}"; } if ($ShowAuthenticatedUsers =~ /B/i) { print "".Format_Bytes($_login_k{$key}).""; } if ($ShowAuthenticatedUsers =~ /L/i) { print "".($_login_l{$key}?Format_Date($_login_l{$key},1):'-').""; } print "\n"; $total_p += $_login_p{$key}||0; $total_h += $_login_h{$key}; $total_k += $_login_k{$key}||0; $count++; } if ($Debug) { debug("Total real / shown : $TotalPages / $total_p - $TotalHits / $total_h - $TotalBytes / $total_h",2); } $rest_p=$TotalPages-$total_p; $rest_h=$TotalHits-$total_h; $rest_k=$TotalBytes-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other logins and/or anonymous print "$Message[125]"; &ShowUserInfo(''); if ($ShowAuthenticatedUsers =~ /P/i) { print "".($rest_p?$rest_p:" ").""; } if ($ShowAuthenticatedUsers =~ /H/i) { print "$rest_h"; } if ($ShowAuthenticatedUsers =~ /B/i) { print "".Format_Bytes($rest_k).""; } if ($ShowAuthenticatedUsers =~ /L/i) { print " "; } print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'allrobots'} || $HTMLOutput{'lastrobots'}) { print "$Center 
\n"; my $title=''; if ($HTMLOutput{'allrobots'}) { $title.="$Message[53]"; } if ($HTMLOutput{'lastrobots'}) { $title.="$Message[9]"; } &tab_head("$title",19,0,'robots'); print "".(scalar keys %_robot_h)." $Message[51]"; if ($ShowRobotsStats =~ /H/i) { print "$Message[57]"; } if ($ShowRobotsStats =~ /B/i) { print "$Message[75]"; } if ($ShowRobotsStats =~ /L/i) { print "$Message[9]"; } print "\n"; $total_p=$total_h=$total_k=$total_r=0; my $count=0; if ($HTMLOutput{'allrobots'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Robot'},\%_robot_h,\%_robot_h); } if ($HTMLOutput{'lastrobots'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Robot'},\%_robot_h,\%_robot_l); } foreach my $key (@keylist) { print "".($RobotsHashIDLib{$key}?$RobotsHashIDLib{$key}:$key).""; if ($ShowRobotsStats =~ /H/i) { print "".($_robot_h{$key}-$_robot_r{$key}).($_robot_r{$key}?"+$_robot_r{$key}":"").""; } if ($ShowRobotsStats =~ /B/i) { print "".Format_Bytes($_robot_k{$key}).""; } if ($ShowRobotsStats =~ /L/i) { print "".($_robot_l{$key}?Format_Date($_robot_l{$key},1):'-').""; } print "\n"; #$total_p += $_robot_p{$key}||0; $total_h += $_robot_h{$key}; $total_k += $_robot_k{$key}||0; $total_r += $_robot_r{$key}||0; $count++; } # For bots we need to count Totals my $TotalPagesRobots = 0; #foreach (values %_robot_p) { $TotalPagesRobots+=$_; } my $TotalHitsRobots = 0; foreach (values %_robot_h) { $TotalHitsRobots+=$_; } my $TotalBytesRobots = 0; foreach (values %_robot_k) { $TotalBytesRobots+=$_; } my $TotalRRobots = 0; foreach (values %_robot_r) { $TotalRRobots+=$_; } $rest_p=0; #$rest_p=$TotalPagesRobots-$total_p; $rest_h=$TotalHitsRobots-$total_h; $rest_k=$TotalBytesRobots-$total_k; $rest_r=$TotalRRobots-$total_r; if ($Debug) { debug("Total real / shown : $TotalPagesRobots / $total_p - $TotalHitsRobots / $total_h - $TotalBytesRobots / $total_k",2); } if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0 || $rest_r > 0) { # All other robots print "$Message[2]"; if ($ShowRobotsStats =~ /H/i) { print "$rest_h"; } if ($ShowRobotsStats =~ /B/i) { print "".(Format_Bytes($rest_k)).""; } if ($ShowRobotsStats =~ /L/i) { print " "; } print "\n"; } &tab_end("* $Message[156]".($TotalRRobots?" $Message[157]":"")); &html_end(1); } if ($HTMLOutput{'urldetail'} || $HTMLOutput{'urlentry'} || $HTMLOutput{'urlexit'}) { # Call to plugins' function ShowPagesFilter foreach my $pluginname (keys %{$PluginsLoaded{'ShowPagesFilter'}}) { my $function="ShowPagesFilter_$pluginname()"; eval("$function"); } print "$Center 
\n"; # Show filter form &ShowFormFilter("urlfilter",$FilterIn{'url'},$FilterEx{'url'}); # Show URL list my $title=''; my $cpt=0; if ($HTMLOutput{'urldetail'}) { $title=$Message[19]; $cpt=(scalar keys %_url_p); } if ($HTMLOutput{'urlentry'}) { $title=$Message[104]; $cpt=(scalar keys %_url_e); } if ($HTMLOutput{'urlexit'}) { $title=$Message[116]; $cpt=(scalar keys %_url_x); } &tab_head("$title",19,0,'urls'); print ""; if ($FilterIn{'url'} || $FilterEx{'url'}) { if ($FilterIn{'url'}) { print "$Message[79] $FilterIn{'url'}"; } if ($FilterIn{'url'} && $FilterEx{'url'}) { print " - "; } if ($FilterEx{'url'}) { print "Exclude $Message[79] $FilterEx{'url'}"; } if ($FilterIn{'url'} || $FilterEx{'url'}) { print ": "; } print "$cpt $Message[28]"; if ($MonthRequired ne 'all') { if ($HTMLOutput{'urldetail'}) { print "
$Message[102]: $TotalDifferentPages $Message[28]"; } } } else { print "$Message[102]: $cpt $Message[28]"; } print ""; if ($ShowPagesStats =~ /P/i) { print "$Message[29]"; } if ($ShowPagesStats =~ /B/i) { print "$Message[106]"; } if ($ShowPagesStats =~ /E/i) { print "$Message[104]"; } if ($ShowPagesStats =~ /X/i) { print "$Message[116]"; } # Call to plugins' function ShowPagesAddField foreach my $pluginname (keys %{$PluginsLoaded{'ShowPagesAddField'}}) { my $function="ShowPagesAddField_$pluginname('title')"; eval("$function"); } print " \n"; $total_p=$total_k=$total_e=$total_x=0; my $count=0; if ($HTMLOutput{'urlentry'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'File'},\%_url_e,\%_url_e); } elsif ($HTMLOutput{'urlexit'}) { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'File'},\%_url_x,\%_url_x); } else { &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'File'},\%_url_p,\%_url_p); } $max_p=1; $max_k=1; foreach my $key (@keylist) { if ($_url_p{$key} > $max_p) { $max_p = $_url_p{$key}; } if ($_url_k{$key}/($_url_p{$key}||1) > $max_k) { $max_k = $_url_k{$key}/($_url_p{$key}||1); } } foreach my $key (@keylist) { print ""; &ShowURLInfo($key); print ""; my $bredde_p=0; my $bredde_e=0; my $bredde_x=0; my $bredde_k=0; if ($max_p > 0) { $bredde_p=int($BarWidth*($_url_p{$key}||0)/$max_p)+1; } if (($bredde_p==1) && $_url_p{$key}) { $bredde_p=2; } if ($max_p > 0) { $bredde_e=int($BarWidth*($_url_e{$key}||0)/$max_p)+1; } if (($bredde_e==1) && $_url_e{$key}) { $bredde_e=2; } if ($max_p > 0) { $bredde_x=int($BarWidth*($_url_x{$key}||0)/$max_p)+1; } if (($bredde_x==1) && $_url_x{$key}) { $bredde_x=2; } if ($max_k > 0) { $bredde_k=int($BarWidth*(($_url_k{$key}||0)/($_url_p{$key}||1))/$max_k)+1; } if (($bredde_k==1) && $_url_k{$key}) { $bredde_k=2; } if ($ShowPagesStats =~ /P/i) { print "$_url_p{$key}"; } if ($ShowPagesStats =~ /B/i) { print "".($_url_k{$key}?Format_Bytes($_url_k{$key}/($_url_p{$key}||1)):" ").""; } if ($ShowPagesStats =~ /E/i) { print "".($_url_e{$key}?$_url_e{$key}:" ").""; } if ($ShowPagesStats =~ /X/i) { print "".($_url_x{$key}?$_url_x{$key}:" ").""; } # Call to plugins' function ShowPagesAddField foreach my $pluginname (keys %{$PluginsLoaded{'ShowPagesAddField'}}) { my $function="ShowPagesAddField_$pluginname('$key')"; eval("$function"); } print ""; # alt and title are not provided to reduce page size if ($ShowPagesStats =~ /P/i) { print "
"; } if ($ShowPagesStats =~ /B/i) { print "
"; } if ($ShowPagesStats =~ /E/i) { print "
"; } if ($ShowPagesStats =~ /X/i) { print ""; } print "\n"; $total_p += $_url_p{$key}; $total_e += $_url_e{$key}; $total_x += $_url_x{$key}; $total_k += $_url_k{$key}; $count++; } if ($Debug) { debug("Total real / shown : $TotalPages / $total_p - $TotalEntries / $total_e - $TotalExits / $total_x - $TotalBytesPages / $total_k",2); } $rest_p=$TotalPages-$total_p; $rest_k=$TotalBytesPages-$total_k; $rest_e=$TotalEntries-$total_e; $rest_x=$TotalExits-$total_x; if ($rest_p > 0 || $rest_e > 0 || $rest_k > 0) { print "$Message[2]"; if ($ShowPagesStats =~ /P/i) { print "".($rest_p?$rest_p:" ").""; } if ($ShowPagesStats =~ /B/i) { print "".($rest_k?Format_Bytes($rest_k/($rest_p||1)):" ").""; } if ($ShowPagesStats =~ /E/i) { print "".($rest_e?$rest_e:" ").""; } if ($ShowPagesStats =~ /X/i) { print "".($rest_x?$rest_x:" ").""; } # Call to plugins' function ShowPagesAddField foreach my $pluginname (keys %{$PluginsLoaded{'ShowPagesAddField'}}) { my $function="ShowPagesAddField_$pluginname('')"; eval("$function"); } print " \n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'unknownos'}) { print "$Center 
\n"; my $title="$Message[46]"; &tab_head("$title",19,0,'unknownos'); print "User agent (".(scalar keys %_unknownreferer_l).")$Message[9]\n"; $total_l=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,1,\%_unknownreferer_l,\%_unknownreferer_l); foreach my $key (@keylist) { my $useragent=CleanFromCSSA($key); print "$useragent"; print "".Format_Date($_unknownreferer_l{$key},1).""; print "\n"; $total_l+=1; $count++; } $rest_l=(scalar keys %_unknownreferer_l)-$total_l; if ($rest_l > 0) { print "$Message[2]"; print "-"; print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'unknownbrowser'}) { print "$Center 
\n"; my $title="$Message[50]"; &tab_head("$title",19,0,'unknownbrowser'); print "User agent (".(scalar keys %_unknownrefererbrowser_l).")$Message[9]\n"; $total_l=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,1,\%_unknownrefererbrowser_l,\%_unknownrefererbrowser_l); foreach my $key (@keylist) { my $useragent=CleanFromCSSA($key); print "$useragent".Format_Date($_unknownrefererbrowser_l{$key},1)."\n"; $total_l+=1; $count++; } $rest_l=(scalar keys %_unknownrefererbrowser_l)-$total_l; if ($rest_l > 0) { print "$Message[2]"; print "-"; print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'osdetail'}) { # Show os versions print "$Center 
"; my $title="$Message[59]"; &tab_head("$title",19,0,'osversions'); print "$Message[58]"; print "$Message[57]$Message[15]"; print " "; print "\n"; $total_h=0; my $count=0; &BuildKeyList(MinimumButNoZero(scalar keys %_os_h,500),1,\%_os_h,\%_os_h); my %keysinkeylist=(); $max_h=1; # Count total by family my %totalfamily_h=(); my $TotalFamily=0; OSLOOP: foreach my $key (@keylist) { $total_h+=$_os_h{$key}; if ($_os_h{$key} > $max_h) { $max_h = $_os_h{$key}; } foreach my $family (@OSFamily) { if ($key =~ /^$family/i) { $totalfamily_h{$family}+=$_os_h{$key}; $TotalFamily+=$_os_h{$key}; next OSLOOP; } } } # Write records grouped in a browser family foreach my $family (@OSFamily) { my $p=' '; if ($total_h) { $p=int($totalfamily_h{$family}/$total_h*1000)/10; $p="$p %"; } my $familyheadershown=0; foreach my $key (reverse sort keys %_os_h) { if ($key =~ /^$family(.*)/i) { if (! $familyheadershown) { print "".uc($family).""; print "".int($totalfamily_h{$family})."$p "; print "\n"; $familyheadershown=1; } $keysinkeylist{$key}=1; my $ver=$1; my $p=' '; if ($total_h) { $p=int($_os_h{$key}/$total_h*1000)/10; $p="$p %"; } print ""; print ""; print "$OSHashLib{$key}"; my $bredde_h=0; if ($max_h > 0) { $bredde_h=int($BarWidth*($_os_h{$key}||0)/$max_h)+1; } if (($bredde_h==1) && $_os_h{$key}) { $bredde_h=2; } print "$_os_h{$key}$p"; print ""; # alt and title are not provided to reduce page size if ($ShowOSStats) { print "
"; } print ""; print "\n"; $count++; } } } # Write other records my $familyheadershown=0; foreach my $key (@keylist) { if ($keysinkeylist{$key}) { next; } if (! $familyheadershown) { my $p=' '; if ($total_h) { $p=int(($total_h-$TotalFamily)/$total_h*1000)/10; $p="$p %"; } print "".uc($Message[2]).""; print "".($total_h-$TotalFamily)."$p "; print "\n"; $familyheadershown=1; } my $p=' '; if ($total_h) { $p=int($_os_h{$key}/$total_h*1000)/10; $p="$p %"; } print ""; if ($key eq 'Unknown') { print "$Message[0]"; } else { my $keywithoutcumul=$key; $keywithoutcumul =~ s/cumul$//i; my $libos=$OSHashLib{$keywithoutcumul}||$keywithoutcumul; my $nameicon=$keywithoutcumul; $nameicon =~ s/[^\w]//g; print "$libos"; } my $bredde_h=0; if ($max_h > 0) { $bredde_h=int($BarWidth*($_os_h{$key}||0)/$max_h)+1; } if (($bredde_h==1) && $_os_h{$key}) { $bredde_h=2; } print "$_os_h{$key}$p"; print ""; # alt and title are not provided to reduce page size if ($ShowOSStats) { print "
"; } print ""; print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'browserdetail'}) { # Show browsers versions print "$Center 
"; my $title="$Message[21]"; &tab_head("$title",19,0,'browsersversions'); print "$Message[58]"; print "$Message[111]$Message[57]$Message[15]"; print " "; print "\n"; $total_h=0; my $count=0; &BuildKeyList(MinimumButNoZero(scalar keys %_browser_h,500),1,\%_browser_h,\%_browser_h); my %keysinkeylist=(); $max_h=1; # Count total by family my %totalfamily_h=(); my $TotalFamily=0; BROWSERLOOP: foreach my $key (@keylist) { $total_h+=$_browser_h{$key}; if ($_browser_h{$key} > $max_h) { $max_h = $_browser_h{$key}; } foreach my $family (keys %BrowsersFamily) { if ($key =~ /^$family/i) { $totalfamily_h{$family}+=$_browser_h{$key}; $TotalFamily+=$_browser_h{$key}; next BROWSERLOOP; } } } # Write records grouped in a browser family foreach my $family (sort { $BrowsersFamily{$a} <=> $BrowsersFamily{$b} } keys %BrowsersFamily) { my $p=' '; if ($total_h) { $p=int($totalfamily_h{$family}/$total_h*1000)/10; $p="$p %"; } my $familyheadershown=0; foreach my $key (reverse sort keys %_browser_h) { if ($key =~ /^$family(.*)/i) { if (! $familyheadershown) { print "".uc($family).""; print " ".int($totalfamily_h{$family})."$p "; print "\n"; $familyheadershown=1; } $keysinkeylist{$key}=1; my $ver=$1; my $p=' '; if ($total_h) { $p=int($_browser_h{$key}/$total_h*1000)/10; $p="$p %"; } print ""; print ""; print "".ucfirst($family)." ".($ver?"$ver":"?").""; print "".($BrowsersHereAreGrabbers{$family}?"$Message[112]":"$Message[113]").""; my $bredde_h=0; if ($max_h > 0) { $bredde_h=int($BarWidth*($_browser_h{$key}||0)/$max_h)+1; } if (($bredde_h==1) && $_browser_h{$key}) { $bredde_h=2; } print "$_browser_h{$key}$p"; print ""; # alt and title are not provided to reduce page size if ($ShowBrowsersStats) { print "
"; } print ""; print "\n"; $count++; } } } # Write other records my $familyheadershown=0; foreach my $key (@keylist) { if ($keysinkeylist{$key}) { next; } if (! $familyheadershown) { my $p=' '; if ($total_h) { $p=int(($total_h-$TotalFamily)/$total_h*1000)/10; $p="$p %"; } print "".uc($Message[2]).""; print " ".($total_h-$TotalFamily)."$p "; print "\n"; $familyheadershown=1; } my $p=' '; if ($total_h) { $p=int($_browser_h{$key}/$total_h*1000)/10; $p="$p %"; } print ""; if ($key eq 'Unknown') { print "$Message[0]?"; } else { my $keywithoutcumul=$key; $keywithoutcumul =~ s/cumul$//i; my $libbrowser=$BrowsersHashIDLib{$keywithoutcumul}||$keywithoutcumul; my $nameicon=$BrowsersHashIcon{$keywithoutcumul}||"notavailable"; print "$libbrowser".($BrowsersHereAreGrabbers{$key}?"$Message[112]":"$Message[113]").""; } my $bredde_h=0; if ($max_h > 0) { $bredde_h=int($BarWidth*($_browser_h{$key}||0)/$max_h)+1; } if (($bredde_h==1) && $_browser_h{$key}) { $bredde_h=2; } print "$_browser_h{$key}$p"; print ""; # alt and title are not provided to reduce page size if ($ShowBrowsersStats) { print "
"; } print ""; print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'refererse'}) { print "$Center 
\n"; my $title="$Message[40]"; &tab_head("$title",19,0,'refererse'); print "$TotalDifferentSearchEngines $Message[122]"; print "$Message[56]$Message[15]"; print "$Message[57]$Message[15]"; print "\n"; $total_s=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Refer'},\%_se_referrals_h,((scalar keys %_se_referrals_p)?\%_se_referrals_p:\%_se_referrals_h)); # before 5.4 only hits were recorded foreach my $key (@keylist) { my $newreferer=CleanFromCSSA($SearchEnginesHashLib{$key}||$key); my $p_p; my $p_h; if ($TotalSearchEnginesPages) { $p_p=int($_se_referrals_p{$key}/$TotalSearchEnginesPages*1000)/10; } if ($TotalSearchEnginesHits) { $p_h=int($_se_referrals_h{$key}/$TotalSearchEnginesHits*1000)/10; } print "$newreferer"; print "".($_se_referrals_p{$key}?$_se_referrals_p{$key}:' ').""; print "".($_se_referrals_p{$key}?"$p_p %":' ').""; print "$_se_referrals_h{$key}"; print "$p_h %"; print "\n"; $total_p += $_se_referrals_p{$key}; $total_h += $_se_referrals_h{$key}; $count++; } if ($Debug) { debug("Total real / shown : $TotalSearchEnginesPages / $total_p - $TotalSearchEnginesHits / $total_h",2); } $rest_p=$TotalSearchEnginesPages-$total_p; $rest_h=$TotalSearchEnginesHits-$total_h; if ($rest_p > 0 || $rest_h > 0) { my $p_p;my $p_h; if ($TotalSearchEnginesPages) { $p_p=int($rest_p/$TotalSearchEnginesPages*1000)/10; } if ($TotalSearchEnginesHits) { $p_h=int($rest_h/$TotalSearchEnginesHits*1000)/10; } print "$Message[2]"; print "".($rest_p?$rest_p:' ').""; print "".($rest_p?"$p_p %":' ').""; print "$rest_h"; print "$p_h %"; print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'refererpages'}) { print "$Center 
\n"; # Show filter form &ShowFormFilter("refererpagesfilter",$FilterIn{'refererpages'},$FilterEx{'refererpages'}); my $title="$Message[41]"; my $cpt=0; $cpt=(scalar keys %_pagesrefs_h); &tab_head("$title",19,0,'refererpages'); print ""; if ($FilterIn{'refererpages'} || $FilterEx{'refererpages'}) { if ($FilterIn{'refererpages'}) { print "$Message[79] $FilterIn{'refererpages'}"; } if ($FilterIn{'refererpages'} && $FilterEx{'refererpages'}) { print " - "; } if ($FilterEx{'refererpages'}) { print "Exclude $Message[79] $FilterEx{'refererpages'}"; } if ($FilterIn{'refererpages'} || $FilterEx{'refererpages'}) { print ": "; } print "$cpt $Message[28]"; #if ($MonthRequired ne 'all') { # if ($HTMLOutput{'refererpages'}) { print "
$Message[102]: $TotalDifferentPages $Message[28]"; } #} } else { print "$Message[102]: $cpt $Message[28]"; } print ""; print "$Message[56]$Message[15]"; print "$Message[57]$Message[15]"; print "\n"; $total_s=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Refer'},\%_pagesrefs_h,((scalar keys %_pagesrefs_p)?\%_pagesrefs_p:\%_pagesrefs_h)); foreach my $key (@keylist) { my $nompage=CleanFromCSSA($key); if (length($nompage)>$MaxLengthOfShownURL) { $nompage=substr($nompage,0,$MaxLengthOfShownURL)."..."; } my $p_p; my $p_h; if ($TotalRefererPages) { $p_p=int($_pagesrefs_p{$key}/$TotalRefererPages*1000)/10; } if ($TotalRefererHits) { $p_h=int($_pagesrefs_h{$key}/$TotalRefererHits*1000)/10; } print ""; &ShowURLInfo($key); print ""; print "".($_pagesrefs_p{$key}?$_pagesrefs_p{$key}:' ')."".($_pagesrefs_p{$key}?"$p_p %":' ').""; print "".($_pagesrefs_h{$key}?$_pagesrefs_h{$key}:' ')."".($_pagesrefs_h{$key}?"$p_h %":' ').""; print "\n"; $total_p += $_pagesrefs_p{$key}; $total_h += $_pagesrefs_h{$key}; $count++; } if ($Debug) { debug("Total real / shown : $TotalRefererPages / $total_p - $TotalRefererHits / $total_h",2); } $rest_p=$TotalRefererPages-$total_p; $rest_h=$TotalRefererHits-$total_h; if ($rest_p > 0 || $rest_h > 0) { my $p_p; my $p_h; if ($TotalRefererPages) { $p_p=int($rest_p/$TotalRefererPages*1000)/10; } if ($TotalRefererHits) { $p_h=int($rest_h/$TotalRefererHits*1000)/10; } print "$Message[2]"; print "".($rest_p?$rest_p:' ').""; print "".($rest_p?"$p_p %":' ').""; print "$rest_h"; print "$p_h %"; print "\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'keyphrases'}) { print "$Center 
\n"; &tab_head($Message[43],19,0,'keyphrases'); print "$TotalDifferentKeyphrases $Message[103]$Message[14]$Message[15]\n"; $total_s=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Keyphrase'},\%_keyphrases,\%_keyphrases); foreach my $key (@keylist) { my $mot; # Convert coded keywords (utf8,...) to be correctly reported in HTML page. if ($PluginsLoaded{'DecodeKey'}{'decodeutfkeys'}) { $mot=CleanFromCSSA(DecodeKey_decodeutfkeys($key,$PageCode||'iso-8859-1')); } else { $mot = CleanFromCSSA(DecodeEncodedString($key)); } my $p; if ($TotalKeyphrases) { $p=int($_keyphrases{$key}/$TotalKeyphrases*1000)/10; } print "".XMLEncode($mot)."$_keyphrases{$key}$p %\n"; $total_s += $_keyphrases{$key}; $count++; } if ($Debug) { debug("Total real / shown : $TotalKeyphrases / $total_s",2); } $rest_s=$TotalKeyphrases-$total_s; if ($rest_s > 0) { my $p; if ($TotalKeyphrases) { $p=int($rest_s/$TotalKeyphrases*1000)/10; } print "$Message[124]$rest_s"; print "$p %\n"; } &tab_end(); &html_end(1); } if ($HTMLOutput{'keywords'}) { print "$Center 
\n"; &tab_head($Message[44],19,0,'keywords'); print "$TotalDifferentKeywords $Message[13]$Message[14]$Message[15]\n"; $total_s=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,$MinHit{'Keyword'},\%_keywords,\%_keywords); foreach my $key (@keylist) { my $mot; # Convert coded keywords (utf8,...) to be correctly reported in HTML page. if ($PluginsLoaded{'DecodeKey'}{'decodeutfkeys'}) { $mot=CleanFromCSSA(DecodeKey_decodeutfkeys($key,$PageCode||'iso-8859-1')); } else { $mot = CleanFromCSSA(DecodeEncodedString($key)); } my $p; if ($TotalKeywords) { $p=int($_keywords{$key}/$TotalKeywords*1000)/10; } print "".XMLEncode($mot)."$_keywords{$key}$p %\n"; $total_s += $_keywords{$key}; $count++; } if ($Debug) { debug("Total real / shown : $TotalKeywords / $total_s",2); } $rest_s=$TotalKeywords-$total_s; if ($rest_s > 0) { my $p; if ($TotalKeywords) { $p=int($rest_s/$TotalKeywords*1000)/10; } print "$Message[30]$rest_s"; print "$p %\n"; } &tab_end(); &html_end(1); } foreach my $code (keys %TrapInfosForHTTPErrorCodes) { if ($HTMLOutput{"errors$code"}) { print "$Center 
\n"; &tab_head($Message[47],19,0,"errors$code"); print "URL (".(scalar keys %_sider404_h).")$Message[49]$Message[23]\n"; $total_h=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,1,\%_sider404_h,\%_sider404_h); foreach my $key (@keylist) { my $nompage=XMLEncode(CleanFromCSSA($key)); #if (length($nompage)>$MaxLengthOfShownURL) { $nompage=substr($nompage,0,$MaxLengthOfShownURL)."..."; } my $referer=XMLEncode(CleanFromCSSA($_referer404_h{$key})); print "$nompage"; print "$_sider404_h{$key}"; print "".($referer?"$referer":" ").""; print "\n"; $total_s += $_sider404_h{$key}; $count++; } # TODO Build TotalErrorHits # if ($Debug) { debug("Total real / shown : $TotalErrorHits / $total_h",2); } # $rest_h=$TotalErrorHits-$total_h; # if ($rest_h > 0) { # my $p; # if ($TotalErrorHits) { $p=int($rest_h/$TotalErrorHits*1000)/10; } # print "$Message[30]"; # print "$rest_h"; # print "..."; # print "\n"; # } &tab_end(); &html_end(1); } } if ($HTMLOutput{'info'}) { # Not yet available print "$Center 
"; &html_end(1); } my $htmloutput=''; foreach my $key (keys %HTMLOutput) { $htmloutput=$key; } if ($htmloutput =~ /^plugin_(\w+)$/) { my $pluginname=$1; print "$Center 
"; my $function="AddHTMLGraph_$pluginname()"; eval("$function"); &html_end(1); } } # Output main page #----------------- if ($HTMLOutput{'main'}) { # SUMMARY #--------------------------------------------------------------------- if ($ShowMonthStats) { if ($Debug) { debug("ShowSummary",2); } #print "$Center 
\n"; my $title="$Message[128]"; &tab_head("$title",0,0,'month'); my $NewLinkParams=${QueryString}; $NewLinkParams =~ s/(^|&)update(=\w*|$)//i; $NewLinkParams =~ s/(^|&)staticlinks(=\w*|$)//i; $NewLinkParams =~ s/(^|&)year=[^&]*//i; $NewLinkParams =~ s/(^|&)month=[^&]*//i; $NewLinkParams =~ s/(^|&)framename=[^&]*//i; $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; if ($NewLinkParams) { $NewLinkParams="${NewLinkParams}&"; } my $NewLinkTarget=''; if ($FrameName eq 'mainright') { $NewLinkTarget=" target=\"_parent\""; } # Ratio my $RatioVisits=0; my $RatioPages=0; my $RatioHits=0; my $RatioBytes=0; if ($TotalUnique > 0) { $RatioVisits=int($TotalVisits/$TotalUnique*100)/100; } if ($TotalVisits > 0) { $RatioPages=int($TotalPages/$TotalVisits*100)/100; } if ($TotalVisits > 0) { $RatioHits=int($TotalHits/$TotalVisits*100)/100; } if ($TotalVisits > 0) { $RatioBytes=int(($TotalBytes/1024)*100/($LogType eq 'M'?$TotalHits:$TotalVisits))/100; } my $colspan=5; my $w='20'; if ($LogType eq 'W' || $LogType eq 'S') { $w='17'; $colspan=6; } # Show first/last print ""; print "$Message[133]\n"; print ($MonthRequired eq 'all'?"$Message[6] $YearRequired":"$Message[5] ".$MonthNumLib{$MonthRequired}." $YearRequired"); print "\n"; print ""; print "$Message[8]\n"; print "".($FirstTime?Format_Date($FirstTime,0):"NA").""; print "\n"; print ""; print "$Message[9]\n"; print "".($LastTime?Format_Date($LastTime,0):"NA")."\n"; print "\n"; # Show main indicators title row print ""; if ($LogType eq 'W' || $LogType eq 'S') { print " "; } if ($ShowMonthStats =~ /U/i) { print "$Message[11]"; } else { print " "; } if ($ShowMonthStats =~ /V/i) { print "$Message[10]"; } else { print " "; } if ($ShowMonthStats =~ /P/i) { print "$Message[56]"; } else { print " "; } if ($ShowMonthStats =~ /H/i) { print "$Message[57]"; } else { print " "; } if ($ShowMonthStats =~ /B/i) { print "$Message[75]"; } else { print " "; } print "\n"; # Show main indicators values for viewed traffic print ""; if ($LogType eq 'M') { print "$Message[165]"; print " 
 \n"; print " 
 \n"; if ($ShowMonthStats =~ /H/i) { print "$TotalHits".($LogType eq 'M'?"":"
($RatioHits ".lc($Message[57]."/".$Message[12]).")"); } else { print " "; } if ($ShowMonthStats =~ /B/i) { print "".Format_Bytes(int($TotalBytes))."
($RatioBytes $Message[108]/".lc($Message[($LogType eq 'M'?149:12)]).")"; } else { print " "; } } else { if ($LogType eq 'W' || $LogType eq 'S') { print "$Message[160] *"; } if ($ShowMonthStats =~ /U/i) { print "".($MonthRequired eq 'all'?"<= $TotalUnique
$Message[129]":"$TotalUnique
 ").""; } else { print " "; } if ($ShowMonthStats =~ /V/i) { print "$TotalVisits
($RatioVisits $Message[52])"; } else { print " "; } if ($ShowMonthStats =~ /P/i) { print "$TotalPages
($RatioPages ".lc($Message[56]."/".$Message[12]).")"; } else { print " "; } if ($ShowMonthStats =~ /H/i) { print "$TotalHits".($LogType eq 'M'?"":"
($RatioHits ".lc($Message[57]."/".$Message[12]).")"); } else { print " "; } if ($ShowMonthStats =~ /B/i) { print "".Format_Bytes(int($TotalBytes))."
($RatioBytes $Message[108]/".lc($Message[($LogType eq 'M'?149:12)]).")"; } else { print " "; } } print "\n"; # Show main indicators values for not viewed traffic values if ($LogType eq 'M' || $LogType eq 'W' || $LogType eq 'S') { print ""; if ($LogType eq 'M') { print "$Message[166]"; print " 
 \n"; print " 
 \n"; if ($ShowMonthStats =~ /H/i) { print "$TotalNotViewedHits"; } else { print " "; } if ($ShowMonthStats =~ /B/i) { print "".Format_Bytes(int($TotalNotViewedBytes)).""; } else { print " "; } } else { if ($LogType eq 'W' || $LogType eq 'S') { print "$Message[161] *"; } print " 
 \n"; if ($ShowMonthStats =~ /P/i) { print "$TotalNotViewedPages"; } else { print " "; } if ($ShowMonthStats =~ /H/i) { print "$TotalNotViewedHits"; } else { print " "; } if ($ShowMonthStats =~ /B/i) { print "".Format_Bytes(int($TotalNotViewedBytes)).""; } else { print " "; } } print "\n"; } &tab_end($LogType eq 'W' || $LogType eq 'S'?"* $Message[159]":""); } # BY MONTH #--------------------------------------------------------------------- if ($ShowMonthStats) { if ($Debug) { debug("ShowMonthStats",2); } print "$Center 
\n"; my $title="$Message[162]"; &tab_head("$title",0,0,'month'); print "\n"; print "
\n"; $average_nb=$average_u=$average_v=$average_p=$average_h=$average_k=0; $total_u=$total_v=$total_p=$total_h=$total_k=0; $max_v=$max_p=$max_h=$max_k=1; # Define total and max for (my $ix=1; $ix<=12; $ix++) { my $monthix=sprintf("%02s",$ix); $total_u+=$MonthUnique{$YearRequired.$monthix}||0; $total_v+=$MonthVisits{$YearRequired.$monthix}||0; $total_p+=$MonthPages{$YearRequired.$monthix}||0; $total_h+=$MonthHits{$YearRequired.$monthix}||0; $total_k+=$MonthBytes{$YearRequired.$monthix}||0; #if (($MonthUnique{$YearRequired.$monthix}||0) > $max_v) { $max_v=$MonthUnique{$YearRequired.$monthix}; } if (($MonthVisits{$YearRequired.$monthix}||0) > $max_v) { $max_v=$MonthVisits{$YearRequired.$monthix}; } #if (($MonthPages{$YearRequired.$monthix}||0) > $max_p) { $max_p=$MonthPages{$YearRequired.$monthix}; } if (($MonthHits{$YearRequired.$monthix}||0) > $max_h) { $max_h=$MonthHits{$YearRequired.$monthix}; } if (($MonthBytes{$YearRequired.$monthix}||0) > $max_k) { $max_k=$MonthBytes{$YearRequired.$monthix}; } } # Define average # TODO # Show bars for month if ($PluginsLoaded{'ShowGraph'}{'graphapplet'}) { my @blocklabel=(); for (my $ix=1; $ix<=12; $ix++) { my $monthix=sprintf("%02s",$ix); push @blocklabel,"$MonthNumLib{$monthix}\§$YearRequired"; } my @vallabel=("$Message[11]","$Message[10]","$Message[56]","$Message[57]","$Message[75]"); my @valcolor=("$color_u","$color_v","$color_p","$color_h","$color_k"); my @valmax=($max_v,$max_v,$max_h,$max_h,$max_k); my @valtotal=($total_u,$total_v,$total_p,$total_h,$total_k); my @valaverage=(); #my @valaverage=($average_v,$average_p,$average_h,$average_k); my @valdata=(); my $xx=0; for (my $ix=1; $ix<=12; $ix++) { my $monthix=sprintf("%02s",$ix); $valdata[$xx++]=$MonthUnique{$YearRequired.$monthix}||0; $valdata[$xx++]=$MonthVisits{$YearRequired.$monthix}||0; $valdata[$xx++]=$MonthPages{$YearRequired.$monthix}||0; $valdata[$xx++]=$MonthHits{$YearRequired.$monthix}||0; $valdata[$xx++]=$MonthBytes{$YearRequired.$monthix}||0; } ShowGraph_graphapplet("$title","month",$ShowMonthStats,\@blocklabel,\@vallabel,\@valcolor,\@valmax,\@valtotal,\@valaverage,\@valdata); } else { print "\n"; print ""; print "\n"; for (my $ix=1; $ix<=12; $ix++) { my $monthix=sprintf("%02s",$ix); my $bredde_u=0; my $bredde_v=0;my $bredde_p=0;my $bredde_h=0;my $bredde_k=0; if ($max_v > 0) { $bredde_u=int(($MonthUnique{$YearRequired.$monthix}||0)/$max_v*$BarHeight)+1; } if ($max_v > 0) { $bredde_v=int(($MonthVisits{$YearRequired.$monthix}||0)/$max_v*$BarHeight)+1; } if ($max_h > 0) { $bredde_p=int(($MonthPages{$YearRequired.$monthix}||0)/$max_h*$BarHeight)+1; } if ($max_h > 0) { $bredde_h=int(($MonthHits{$YearRequired.$monthix}||0)/$max_h*$BarHeight)+1; } if ($max_k > 0) { $bredde_k=int(($MonthBytes{$YearRequired.$monthix}||0)/$max_k*$BarHeight)+1; } print "\n"; } print ""; print "\n"; # Show lib for month print ""; # if (!$StaticLinks) { # print ""; # } # else { print ""; # } for (my $ix=1; $ix<=12; $ix++) { my $monthix=sprintf("%02s",$ix); # if (!$StaticLinks) { # print ""; # } # else { print ""; # } } # if (!$StaticLinks) { # print ""; # } # else { print ""; # } print "\n"; print "
 "; if ($ShowMonthStats =~ /U/i) { print ""; } if ($ShowMonthStats =~ /V/i) { print ""; } if ($QueryString !~ /buildpdf/i) { print " "; } if ($ShowMonthStats =~ /P/i) { print ""; } if ($ShowMonthStats =~ /H/i) { print ""; } if ($ShowMonthStats =~ /B/i) { print ""; } print " 
<< $MonthNumLib{$monthix}
$YearRequired		".(! $StaticLinks && $monthix==$nowmonth && $YearRequired==$nowyear?'':''); print "$MonthNumLib{$monthix}
$YearRequired"; print (! $StaticLinks && $monthix==$nowmonth && $YearRequired==$nowyear?'':'')."		>> 
\n"; } print "
\n"; # Show data array for month if ($AddDataArrayMonthStats) { print "\n"; print ""; if ($ShowMonthStats =~ /U/i) { print ""; } if ($ShowMonthStats =~ /V/i) { print ""; } if ($ShowMonthStats =~ /P/i) { print ""; } if ($ShowMonthStats =~ /H/i) { print ""; } if ($ShowMonthStats =~ /B/i) { print ""; } print "\n"; for (my $ix=1; $ix<=12; $ix++) { my $monthix=sprintf("%02s",$ix); print ""; print ""; if ($ShowMonthStats =~ /U/i) { print ""; } if ($ShowMonthStats =~ /V/i) { print ""; } if ($ShowMonthStats =~ /P/i) { print ""; } if ($ShowMonthStats =~ /H/i) { print ""; } if ($ShowMonthStats =~ /B/i) { print ""; } print "\n"; } # Average row # TODO # Total row print ""; if ($ShowMonthStats =~ /U/i) { print ""; } if ($ShowMonthStats =~ /V/i) { print ""; } if ($ShowMonthStats =~ /P/i) { print ""; } if ($ShowMonthStats =~ /H/i) { print ""; } if ($ShowMonthStats =~ /B/i) { print ""; } print "\n"; print "
$Message[5]$Message[11]$Message[10]$Message[56]$Message[57]$Message[75]
".(! $StaticLinks && $monthix==$nowmonth && $YearRequired==$nowyear?'':''); print "$MonthNumLib{$monthix} $YearRequired"; print (! $StaticLinks && $monthix==$nowmonth && $YearRequired==$nowyear?'':'')."",$MonthUnique{$YearRequired.$monthix}?$MonthUnique{$YearRequired.$monthix}:"0","",$MonthVisits{$YearRequired.$monthix}?$MonthVisits{$YearRequired.$monthix}:"0","",$MonthPages{$YearRequired.$monthix}?$MonthPages{$YearRequired.$monthix}:"0","",$MonthHits{$YearRequired.$monthix}?$MonthHits{$YearRequired.$monthix}:"0","",Format_Bytes(int($MonthBytes{$YearRequired.$monthix}||0)),"
$Message[102]$total_u$total_v$total_p$total_h".Format_Bytes($total_k)."
\n
\n"; } print "
\n"; print "\n"; &tab_end(); } print "\n \n\n"; # BY DAY OF MONTH #--------------------------------------------------------------------- if ($ShowDaysOfMonthStats) { if ($Debug) { debug("ShowDaysOfMonthStats",2); } print "$Center 
\n"; my $title="$Message[138]"; &tab_head("$title",0,0,'daysofmonth'); print ""; print "\n"; print "
\n"; my $NewLinkParams=${QueryString}; $NewLinkParams =~ s/(^|&)update(=\w*|$)//i; $NewLinkParams =~ s/(^|&)staticlinks(=\w*|$)//i; $NewLinkParams =~ s/(^|&)year=[^&]*//i; $NewLinkParams =~ s/(^|&)month=[^&]*//i; $NewLinkParams =~ s/(^|&)framename=[^&]*//i; $NewLinkParams =~ tr/&/&/s; $NewLinkParams =~ s/^&//; $NewLinkParams =~ s/&$//; if ($NewLinkParams) { $NewLinkParams="${NewLinkParams}&"; } my $NewLinkTarget=''; if ($FrameName eq 'mainright') { $NewLinkTarget=" target=\"_parent\""; } $average_nb=$average_u=$average_v=$average_p=$average_h=$average_k=0; $total_u=$total_v=$total_p=$total_h=$total_k=0; # Define total and max $max_v=$max_h=$max_k=0; # Start from 0 because can be lower than 1 foreach my $daycursor ($firstdaytoshowtime..$lastdaytoshowtime) { $daycursor =~ /^(\d\d\d\d)(\d\d)(\d\d)/; my $year=$1; my $month=$2; my $day=$3; if (! DateIsValid($day,$month,$year)) { next; } # If not an existing day, go to next $total_v+=$DayVisits{$year.$month.$day}||0; $total_p+=$DayPages{$year.$month.$day}||0; $total_h+=$DayHits{$year.$month.$day}||0; $total_k+=$DayBytes{$year.$month.$day}||0; if (($DayVisits{$year.$month.$day}||0) > $max_v) { $max_v=$DayVisits{$year.$month.$day}; } #if (($DayPages{$year.$month.$day}||0) > $max_p) { $max_p=$DayPages{$year.$month.$day}; } if (($DayHits{$year.$month.$day}||0) > $max_h) { $max_h=$DayHits{$year.$month.$day}; } if (($DayBytes{$year.$month.$day}||0) > $max_k) { $max_k=$DayBytes{$year.$month.$day}; } } # Define average foreach my $daycursor ($firstdaytocountaverage..$lastdaytocountaverage) { $daycursor =~ /^(\d\d\d\d)(\d\d)(\d\d)/; my $year=$1; my $month=$2; my $day=$3; if (! DateIsValid($day,$month,$year)) { next; } # If not an existing day, go to next $average_nb++; # Increase number of day used to count $average_v+=($DayVisits{$daycursor}||0); $average_p+=($DayPages{$daycursor}||0); $average_h+=($DayHits{$daycursor}||0); $average_k+=($DayBytes{$daycursor}||0); } if ($average_nb) { $average_v=$average_v/$average_nb; $average_p=$average_p/$average_nb; $average_h=$average_h/$average_nb; $average_k=$average_k/$average_nb; if ($average_v > $max_v) { $max_v=$average_v; } #if ($average_p > $max_p) { $max_p=$average_p; } if ($average_h > $max_h) { $max_h=$average_h; } if ($average_k > $max_k) { $max_k=$average_k; } } else { $average_v="?"; $average_p="?"; $average_h="?"; $average_k="?"; } # Show bars for day if ($PluginsLoaded{'ShowGraph'}{'graphapplet'}) { my @blocklabel=(); foreach my $daycursor ($firstdaytoshowtime..$lastdaytoshowtime) { $daycursor =~ /^(\d\d\d\d)(\d\d)(\d\d)/; my $year=$1; my $month=$2; my $day=$3; if (! DateIsValid($day,$month,$year)) { next; } # If not an existing day, go to next my $bold=($day==$nowday && $month==$nowmonth && $year==$nowyear?':':''); my $weekend=(DayOfWeek($day,$month,$year)=~/[06]/?'!':''); push @blocklabel,"$day§$MonthNumLib{$month}$weekend$bold"; } my @vallabel=("$Message[10]","$Message[56]","$Message[57]","$Message[75]"); my @valcolor=("$color_v","$color_p","$color_h","$color_k"); my @valmax=($max_v,$max_h,$max_h,$max_k); my @valtotal=($total_v,$total_p,$total_h,$total_k); $average_v=sprintf("%.2f",$average_v); $average_p=sprintf("%.2f",$average_p); $average_h=sprintf("%.2f",$average_h); $average_k=(int($average_k)?Format_Bytes(sprintf("%.2f",$average_k)):"0.00"); my @valaverage=($average_v,$average_p,$average_h,$average_k); my @valdata=(); my $xx=0; foreach my $daycursor ($firstdaytoshowtime..$lastdaytoshowtime) { $daycursor =~ /^(\d\d\d\d)(\d\d)(\d\d)/; my $year=$1; my $month=$2; my $day=$3; if (! DateIsValid($day,$month,$year)) { next; } # If not an existing day, go to next $valdata[$xx++]=$DayVisits{$year.$month.$day}||0; $valdata[$xx++]=$DayPages{$year.$month.$day}||0; $valdata[$xx++]=$DayHits{$year.$month.$day}||0; $valdata[$xx++]=$DayBytes{$year.$month.$day}||0; } ShowGraph_graphapplet("$title","daysofmonth",$ShowDaysOfMonthStats,\@blocklabel,\@vallabel,\@valcolor,\@valmax,\@valtotal,\@valaverage,\@valdata); } else { print "\n"; print "\n"; foreach my $daycursor ($firstdaytoshowtime..$lastdaytoshowtime) { $daycursor =~ /^(\d\d\d\d)(\d\d)(\d\d)/; my $year=$1; my $month=$2; my $day=$3; if (! DateIsValid($day,$month,$year)) { next; } # If not an existing day, go to next my $bredde_v=0; my $bredde_p=0; my $bredde_h=0; my $bredde_k=0; if ($max_v > 0) { $bredde_v=int(($DayVisits{$year.$month.$day}||0)/$max_v*$BarHeight)+1; } if ($max_h > 0) { $bredde_p=int(($DayPages{$year.$month.$day}||0)/$max_h*$BarHeight)+1; } if ($max_h > 0) { $bredde_h=int(($DayHits{$year.$month.$day}||0)/$max_h*$BarHeight)+1; } if ($max_k > 0) { $bredde_k=int(($DayBytes{$year.$month.$day}||0)/$max_k*$BarHeight)+1; } print "\n"; } print ""; # Show average value cell print "\n"; print "\n"; # Show lib for day print ""; foreach my $daycursor ($firstdaytoshowtime..$lastdaytoshowtime) { $daycursor =~ /^(\d\d\d\d)(\d\d)(\d\d)/; my $year=$1; my $month=$2; my $day=$3; if (! DateIsValid($day,$month,$year)) { next; } # If not an existing day, go to next my $dayofweekcursor=DayOfWeek($day,$month,$year); print ""; print (! $StaticLinks && $day==$nowday && $month==$nowmonth && $year==$nowyear?'':''); print "$day
".$MonthNumLib{$month}.""; print (! $StaticLinks && $day==$nowday && $month==$nowmonth && $year==$nowyear?'':''); print "\n"; } print ""; print "\n"; print "\n"; print "
"; if ($ShowDaysOfMonthStats =~ /V/i) { print ""; } if ($ShowDaysOfMonthStats =~ /P/i) { print ""; } if ($ShowDaysOfMonthStats =~ /H/i) { print ""; } if ($ShowDaysOfMonthStats =~ /B/i) { print ""; } print " "; my $bredde_v=0; my $bredde_p=0; my $bredde_h=0; my $bredde_k=0; if ($max_v > 0) { $bredde_v=int($average_v/$max_v*$BarHeight)+1; } if ($max_h > 0) { $bredde_p=int($average_p/$max_h*$BarHeight)+1; } if ($max_h > 0) { $bredde_h=int($average_h/$max_h*$BarHeight)+1; } if ($max_k > 0) { $bredde_k=int($average_k/$max_k*$BarHeight)+1; } $average_v=sprintf("%.2f",$average_v); $average_p=sprintf("%.2f",$average_p); $average_h=sprintf("%.2f",$average_h); $average_k=(int($average_k)?Format_Bytes(sprintf("%.2f",$average_k)):"0.00"); if ($ShowDaysOfMonthStats =~ /V/i) { print ""; } if ($ShowDaysOfMonthStats =~ /P/i) { print ""; } if ($ShowDaysOfMonthStats =~ /H/i) { print ""; } if ($ShowDaysOfMonthStats =~ /B/i) { print ""; } print "
 $Message[96]
\n"; } print "
\n"; # Show data array for days if ($AddDataArrayShowDaysOfMonthStats) { print "\n"; print ""; if ($ShowDaysOfMonthStats =~ /V/i) { print ""; } if ($ShowDaysOfMonthStats =~ /P/i) { print ""; } if ($ShowDaysOfMonthStats =~ /H/i) { print ""; } if ($ShowDaysOfMonthStats =~ /B/i) { print ""; } print ""; foreach my $daycursor ($firstdaytoshowtime..$lastdaytoshowtime) { $daycursor =~ /^(\d\d\d\d)(\d\d)(\d\d)/; my $year=$1; my $month=$2; my $day=$3; if (! DateIsValid($day,$month,$year)) { next; } # If not an existing day, go to next my $dayofweekcursor=DayOfWeek($day,$month,$year); print ""; print ""; if ($ShowDaysOfMonthStats =~ /V/i) { print ""; } if ($ShowDaysOfMonthStats =~ /P/i) { print ""; } if ($ShowDaysOfMonthStats =~ /H/i) { print ""; } if ($ShowDaysOfMonthStats =~ /B/i) { print ""; } print "\n"; } # Average row print ""; if ($ShowDaysOfMonthStats =~ /V/i) { print ""; } if ($ShowDaysOfMonthStats =~ /P/i) { print ""; } if ($ShowDaysOfMonthStats =~ /H/i) { print ""; } if ($ShowDaysOfMonthStats =~ /B/i) { print ""; } print "\n"; # Total row print ""; if ($ShowDaysOfMonthStats =~ /V/i) { print ""; } if ($ShowDaysOfMonthStats =~ /P/i) { print ""; } if ($ShowDaysOfMonthStats =~ /H/i) { print ""; } if ($ShowDaysOfMonthStats =~ /B/i) { print ""; } print "\n"; print "
$Message[4]$Message[10]$Message[56]$Message[57]$Message[75]
".(! $StaticLinks && $day==$nowday && $month==$nowmonth && $year==$nowyear?'':''); print Format_Date("$year$month$day"."000000",2); print (! $StaticLinks && $day==$nowday && $month==$nowmonth && $year==$nowyear?'':'')."",$DayVisits{$year.$month.$day}?$DayVisits{$year.$month.$day}:"0","",$DayPages{$year.$month.$day}?$DayPages{$year.$month.$day}:"0","",$DayHits{$year.$month.$day}?$DayHits{$year.$month.$day}:"0","",Format_Bytes(int($DayBytes{$year.$month.$day}||0)),"
$Message[96]$average_v$average_p$average_h$average_k
$Message[102]$total_v$total_p$total_h".Format_Bytes($total_k)."
\n
"; } print "
\n"; print "\n"; &tab_end(); } # BY DAY OF WEEK #------------------------- if ($ShowDaysOfWeekStats) { if ($Debug) { debug("ShowDaysOfWeekStats",2); } print "$Center 
\n"; my $title="$Message[91]"; &tab_head("$title",18,0,'daysofweek'); print ""; print ""; print "
\n"; $max_h=$max_k=0; # Start from 0 because can be lower than 1 # Get average value for day of week my @avg_dayofweek_nb=(); my @avg_dayofweek_p=(); my @avg_dayofweek_h=(); my @avg_dayofweek_k=(); foreach my $daycursor ($firstdaytocountaverage..$lastdaytocountaverage) { $daycursor =~ /^(\d\d\d\d)(\d\d)(\d\d)/; my $year=$1; my $month=$2; my $day=$3; if (! DateIsValid($day,$month,$year)) { next; } # If not an existing day, go to next my $dayofweekcursor=DayOfWeek($day,$month,$year); $avg_dayofweek_nb[$dayofweekcursor]++; # Increase number of day used to count for this day of week $avg_dayofweek_p[$dayofweekcursor]+=($DayPages{$daycursor}||0); $avg_dayofweek_h[$dayofweekcursor]+=($DayHits{$daycursor}||0); $avg_dayofweek_k[$dayofweekcursor]+=($DayBytes{$daycursor}||0); } for (@DOWIndex) { if ($avg_dayofweek_nb[$_]) { $avg_dayofweek_p[$_]=$avg_dayofweek_p[$_]/$avg_dayofweek_nb[$_]; $avg_dayofweek_h[$_]=$avg_dayofweek_h[$_]/$avg_dayofweek_nb[$_]; $avg_dayofweek_k[$_]=$avg_dayofweek_k[$_]/$avg_dayofweek_nb[$_]; #if ($avg_dayofweek_p[$_] > $max_p) { $max_p = $avg_dayofweek_p[$_]; } if ($avg_dayofweek_h[$_] > $max_h) { $max_h = $avg_dayofweek_h[$_]; } if ($avg_dayofweek_k[$_] > $max_k) { $max_k = $avg_dayofweek_k[$_]; } } else { $avg_dayofweek_p[$_]="?"; $avg_dayofweek_h[$_]="?"; $avg_dayofweek_k[$_]="?"; } } # Show bars for days of week if ($PluginsLoaded{'ShowGraph'}{'graphapplet'}) { my @blocklabel=(); for (@DOWIndex) { push @blocklabel,($Message[$_+84].($_=~/[06]/?"!":"")); } my @vallabel=("$Message[56]","$Message[57]","$Message[75]"); my @valcolor=("$color_p","$color_h","$color_k"); my @valmax=(int($max_h),int($max_h),int($max_k)); my @valtotal=($total_p,$total_h,$total_k); $average_p=sprintf("%.2f",$average_p); $average_h=sprintf("%.2f",$average_h); $average_k=(int($average_k)?Format_Bytes(sprintf("%.2f",$average_k)):"0.00"); my @valaverage=($average_p,$average_h,$average_k); my @valdata=(); my $xx=0; for (@DOWIndex) { $valdata[$xx++]=$avg_dayofweek_p[$_]||0; $valdata[$xx++]=$avg_dayofweek_h[$_]||0; $valdata[$xx++]=$avg_dayofweek_k[$_]||0; # Round to be ready to show array $avg_dayofweek_p[$_]=sprintf("%.2f",$avg_dayofweek_p[$_]); $avg_dayofweek_h[$_]=sprintf("%.2f",$avg_dayofweek_h[$_]); $avg_dayofweek_k[$_]=sprintf("%.2f",$avg_dayofweek_k[$_]); # Remove decimal part that are .0 if ($avg_dayofweek_p[$_] == int($avg_dayofweek_p[$_])) { $avg_dayofweek_p[$_]=int($avg_dayofweek_p[$_]); } if ($avg_dayofweek_h[$_] == int($avg_dayofweek_h[$_])) { $avg_dayofweek_h[$_]=int($avg_dayofweek_h[$_]); } } ShowGraph_graphapplet("$title","daysofweek",$ShowDaysOfWeekStats,\@blocklabel,\@vallabel,\@valcolor,\@valmax,\@valtotal,\@valaverage,\@valdata); } else { print "\n"; print "\n"; for (@DOWIndex) { my $bredde_p=0; my $bredde_h=0; my $bredde_k=0; if ($max_h > 0) { $bredde_p=int($avg_dayofweek_p[$_]/$max_h*$BarHeight)+1; } if ($max_h > 0) { $bredde_h=int($avg_dayofweek_h[$_]/$max_h*$BarHeight)+1; } if ($max_k > 0) { $bredde_k=int($avg_dayofweek_k[$_]/$max_k*$BarHeight)+1; } $avg_dayofweek_p[$_]=sprintf("%.2f",$avg_dayofweek_p[$_]); $avg_dayofweek_h[$_]=sprintf("%.2f",$avg_dayofweek_h[$_]); $avg_dayofweek_k[$_]=sprintf("%.2f",$avg_dayofweek_k[$_]); # Remove decimal part that are .0 if ($avg_dayofweek_p[$_] == int($avg_dayofweek_p[$_])) { $avg_dayofweek_p[$_]=int($avg_dayofweek_p[$_]); } if ($avg_dayofweek_h[$_] == int($avg_dayofweek_h[$_])) { $avg_dayofweek_h[$_]=int($avg_dayofweek_h[$_]); } print "\n"; } print "\n"; print "\n"; for (@DOWIndex) { print "".(! $StaticLinks && $_==($nowwday-1) && $MonthRequired==$nowmonth && $YearRequired==$nowyear?'':''); print $Message[$_+84]; print (! $StaticLinks && $_==($nowwday-1) && $MonthRequired==$nowmonth && $YearRequired==$nowyear?'':'').""; } print "\n
"; if ($ShowDaysOfWeekStats =~ /P/i) { print ""; } if ($ShowDaysOfWeekStats =~ /H/i) { print ""; } if ($ShowDaysOfWeekStats =~ /B/i) { print ""; } print "
\n"; } print "
\n"; # Show data array for days of week if ($AddDataArrayShowDaysOfWeekStats) { print "\n"; print ""; if ($ShowDaysOfWeekStats =~ /P/i) { print ""; } if ($ShowDaysOfWeekStats =~ /H/i) { print ""; } if ($ShowDaysOfWeekStats =~ /B/i) { print ""; } for (@DOWIndex) { print ""; print ""; if ($ShowDaysOfWeekStats =~ /P/i) { print ""; } if ($ShowDaysOfWeekStats =~ /H/i) { print ""; } if ($ShowDaysOfWeekStats =~ /B/i) { print ""; } print "\n"; } print "
$Message[4]$Message[56]$Message[57]$Message[75]
".(! $StaticLinks && $_==($nowwday-1) && $MonthRequired==$nowmonth && $YearRequired==$nowyear?'':''); print $Message[$_+84]; print "",$avg_dayofweek_p[$_],"",$avg_dayofweek_h[$_],"",Format_Bytes($avg_dayofweek_k[$_]),"
\n
\n"; } print "
"; print "\n"; &tab_end(); } # BY HOUR #---------------------------- if ($ShowHoursStats) { if ($Debug) { debug("ShowHoursStats",2); } print "$Center 
\n"; my $title="$Message[20]"; if ($PluginsLoaded{'GetTimeZoneTitle'}{'timezone'}) { $title.=" (GMT ".(GetTimeZoneTitle_timezone()>=0?"+":"").int(GetTimeZoneTitle_timezone()).")"; } &tab_head("$title",19,0,'hours'); print "\n"; print "
\n"; $max_h=$max_k=1; for (my $ix=0; $ix<=23; $ix++) { #if ($_time_p[$ix]>$max_p) { $max_p=$_time_p[$ix]; } if ($_time_h[$ix]>$max_h) { $max_h=$_time_h[$ix]; } if ($_time_k[$ix]>$max_k) { $max_k=$_time_k[$ix]; } } # Show bars for hour if ($PluginsLoaded{'ShowGraph'}{'graphapplet'}) { my @blocklabel=(0..23); my @vallabel=("$Message[56]","$Message[57]","$Message[75]"); my @valcolor=("$color_p","$color_h","$color_k"); my @valmax=(int($max_h),int($max_h),int($max_k)); my @valtotal=($total_p,$total_h,$total_k); my @valaverage=($average_p,$average_h,$average_k); my @valdata=(); my $xx=0; for (0..23) { $valdata[$xx++]=$_time_p[$_]||0; $valdata[$xx++]=$_time_h[$_]||0; $valdata[$xx++]=$_time_k[$_]||0; } ShowGraph_graphapplet("$title","hours",$ShowHoursStats,\@blocklabel,\@vallabel,\@valcolor,\@valmax,\@valtotal,\@valaverage,\@valdata); } else { print "\n"; print "\n"; for (my $ix=0; $ix<=23; $ix++) { my $bredde_p=0;my $bredde_h=0;my $bredde_k=0; if ($max_h > 0) { $bredde_p=int($BarHeight*$_time_p[$ix]/$max_h)+1; } if ($max_h > 0) { $bredde_h=int($BarHeight*$_time_h[$ix]/$max_h)+1; } if ($max_k > 0) { $bredde_k=int($BarHeight*$_time_k[$ix]/$max_k)+1; } print "\n"; } print "\n"; # Show hour lib print ""; for (my $ix=0; $ix<=23; $ix++) { print "\n"; # width=19 instead of 18 to avoid a MacOS browser bug. } print "\n"; # Show clock icon print "\n"; for (my $ix=0; $ix<=23; $ix++) { my $hrs=($ix>=12?$ix-12:$ix); my $hre=($ix>=12?$ix-11:$ix+1); my $apm=($ix>=12?"pm":"am"); print "\n"; } print "\n"; print "
"; if ($ShowHoursStats =~ /P/i) { print ""; } if ($ShowHoursStats =~ /H/i) { print ""; } if ($ShowHoursStats =~ /B/i) { print ""; } print "
$ix
\"$hrs:00
\n"; } print "
\n"; # Show data array for hours if ($AddDataArrayShowHoursStats) { print "\n"; print ""; print ""; print "
\n"; print "\n"; print ""; if ($ShowHoursStats =~ /P/i) { print ""; } if ($ShowHoursStats =~ /H/i) { print ""; } if ($ShowHoursStats =~ /B/i) { print ""; } print ""; for (my $ix=0; $ix<=11; $ix++) { my $monthix=($ix<10?"0$ix":"$ix"); print ""; print ""; if ($ShowHoursStats =~ /P/i) { print ""; } if ($ShowHoursStats =~ /H/i) { print ""; } if ($ShowHoursStats =~ /B/i) { print ""; } print "\n"; } print "
$Message[20]$Message[56]$Message[57]$Message[75]
$monthix",$_time_p[$monthix]?$_time_p[$monthix]:"0","",$_time_h[$monthix]?$_time_h[$monthix]:"0","",Format_Bytes(int($_time_k[$monthix])),"
\n"; print "
 
\n"; print "\n"; print ""; if ($ShowHoursStats =~ /P/i) { print ""; } if ($ShowHoursStats =~ /H/i) { print ""; } if ($ShowHoursStats =~ /B/i) { print ""; } print "\n"; for (my $ix=12; $ix<=23; $ix++) { my $monthix=($ix<10?"0$ix":"$ix"); print ""; print ""; if ($ShowHoursStats =~ /P/i) { print ""; } if ($ShowHoursStats =~ /H/i) { print ""; } if ($ShowHoursStats =~ /B/i) { print ""; } print "\n"; } print "
$Message[20]$Message[56]$Message[57]$Message[75]
$monthix",$_time_p[$monthix]?$_time_p[$monthix]:"0","",$_time_h[$monthix]?$_time_h[$monthix]:"0","",Format_Bytes(int($_time_k[$monthix])),"
\n"; print "
\n"; print "
\n"; } print "
\n"; &tab_end(); } print "\n \n\n"; # BY COUNTRY/DOMAIN #--------------------------- if ($ShowDomainsStats) { if ($Debug) { debug("ShowDomainsStats",2); } print "$Center 
\n"; my $title="$Message[25] ($Message[77] $MaxNbOf{'Domain'})   -   $Message[80]"; &tab_head("$title",19,0,'countries'); print " $Message[17]"; if ($ShowDomainsStats =~ /P/i) { print "$Message[56]"; } if ($ShowDomainsStats =~ /H/i) { print "$Message[57]"; } if ($ShowDomainsStats =~ /B/i) { print "$Message[75]"; } print " "; print "\n"; $total_p=$total_h=$total_k=0; $max_h=1; foreach (values %_domener_h) { if ($_ > $max_h) { $max_h = $_; } } $max_k=1; foreach (values %_domener_k) { if ($_ > $max_k) { $max_k = $_; } } my $count=0; &BuildKeyList($MaxNbOf{'Domain'},$MinHit{'Domain'},\%_domener_h,\%_domener_p); foreach my $key (@keylist) { my $bredde_p=0;my $bredde_h=0;my $bredde_k=0; if ($max_h > 0) { $bredde_p=int($BarWidth*$_domener_p{$key}/$max_h)+1; } # use max_h to enable to compare pages with hits if ($_domener_p{$key} && $bredde_p==1) { $bredde_p=2; } if ($max_h > 0) { $bredde_h=int($BarWidth*$_domener_h{$key}/$max_h)+1; } if ($_domener_h{$key} && $bredde_h==1) { $bredde_h=2; } if ($max_k > 0) { $bredde_k=int($BarWidth*($_domener_k{$key}||0)/$max_k)+1; } if ($_domener_k{$key} && $bredde_k==1) { $bredde_k=2; } my $newkey=lc($key); if ($newkey eq 'ip' || ! $DomainsHashIDLib{$newkey}) { print "$Message[0]$newkey"; } else { print "$DomainsHashIDLib{$newkey}$newkey"; } if ($ShowDomainsStats =~ /P/i) { print "".($_domener_p{$key}?$_domener_p{$key}:' ').""; } if ($ShowDomainsStats =~ /H/i) { print "$_domener_h{$key}"; } if ($ShowDomainsStats =~ /B/i) { print "".Format_Bytes($_domener_k{$key}).""; } print ""; if ($ShowDomainsStats =~ /P/i) { print "
\n"; } if ($ShowDomainsStats =~ /H/i) { print "
\n"; } if ($ShowDomainsStats =~ /B/i) { print ""; } print ""; print "\n"; $total_p += $_domener_p{$key}; $total_h += $_domener_h{$key}; $total_k += $_domener_k{$key}||0; $count++; } $rest_p=$TotalPages-$total_p; $rest_h=$TotalHits-$total_h; $rest_k=$TotalBytes-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other domains (known or not) print " $Message[2]"; if ($ShowDomainsStats =~ /P/i) { print "$rest_p"; } if ($ShowDomainsStats =~ /H/i) { print "$rest_h"; } if ($ShowDomainsStats =~ /B/i) { print "".Format_Bytes($rest_k).""; } print " "; print "\n"; } &tab_end(); } # BY HOST/VISITOR #-------------------------- if ($ShowHostsStats) { if ($Debug) { debug("ShowHostsStats",2); } print "$Center 
\n"; my $title="$Message[81] ($Message[77] $MaxNbOf{'HostsShown'})   -   $Message[80]   -   $Message[9]   -   $Message[45]"; &tab_head("$title",19,0,'visitors'); print ""; print ""; if ($MonthRequired ne 'all') { print "$Message[81] : $TotalHostsKnown $Message[82], $TotalHostsUnknown $Message[1] - $TotalUnique $Message[11]"; } else { print "$Message[81] : ".(scalar keys %_host_h).""; } &ShowHostInfo('__title__'); if ($ShowHostsStats =~ /P/i) { print "$Message[56]"; } if ($ShowHostsStats =~ /H/i) { print "$Message[57]"; } if ($ShowHostsStats =~ /B/i) { print "$Message[75]"; } if ($ShowHostsStats =~ /L/i) { print "$Message[9]"; } print "\n"; $total_p=$total_h=$total_k=0; my $count=0; &BuildKeyList($MaxNbOf{'HostsShown'},$MinHit{'Host'},\%_host_h,\%_host_p); foreach my $key (@keylist) { print ""; print "$key"; &ShowHostInfo($key); if ($ShowHostsStats =~ /P/i) { print "".($_host_p{$key}||" ").""; } if ($ShowHostsStats =~ /H/i) { print "$_host_h{$key}"; } if ($ShowHostsStats =~ /B/i) { print "".Format_Bytes($_host_k{$key}).""; } if ($ShowHostsStats =~ /L/i) { print "".($_host_l{$key}?Format_Date($_host_l{$key},1):'-').""; } print "\n"; $total_p += $_host_p{$key}; $total_h += $_host_h{$key}; $total_k += $_host_k{$key}||0; $count++; } $rest_p=$TotalPages-$total_p; $rest_h=$TotalHits-$total_h; $rest_k=$TotalBytes-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other visitors (known or not) print ""; print "$Message[2]"; &ShowHostInfo(''); if ($ShowHostsStats =~ /P/i) { print "$rest_p"; } if ($ShowHostsStats =~ /H/i) { print "$rest_h"; } if ($ShowHostsStats =~ /B/i) { print "".Format_Bytes($rest_k).""; } if ($ShowHostsStats =~ /L/i) { print " "; } print "\n"; } &tab_end(); } # BY SENDER EMAIL #---------------------------- if ($ShowEMailSenders) { &ShowEmailSendersChart($NewLinkParams,$NewLinkTarget); } # BY RECEIVER EMAIL #---------------------------- if ($ShowEMailReceivers) { &ShowEmailReceiversChart($NewLinkParams,$NewLinkTarget); } # BY LOGIN #---------------------------- if ($ShowAuthenticatedUsers) { if ($Debug) { debug("ShowAuthenticatedUsers",2); } print "$Center 
\n"; my $title="$Message[94] ($Message[77] $MaxNbOf{'LoginShown'})   -   $Message[80]"; if ($ShowAuthenticatedUsers =~ /L/i) { $title.="   -   $Message[9]"; } &tab_head("$title",19,0,'logins'); print "$Message[94] : ".(scalar keys %_login_h).""; &ShowUserInfo('__title__'); if ($ShowAuthenticatedUsers =~ /P/i) { print "$Message[56]"; } if ($ShowAuthenticatedUsers =~ /H/i) { print "$Message[57]"; } if ($ShowAuthenticatedUsers =~ /B/i) { print "$Message[75]"; } if ($ShowAuthenticatedUsers =~ /L/i) { print "$Message[9]"; } print "\n"; $total_p=$total_h=$total_k=0; $max_h=1; foreach (values %_login_h) { if ($_ > $max_h) { $max_h = $_; } } $max_k=1; foreach (values %_login_k) { if ($_ > $max_k) { $max_k = $_; } } my $count=0; &BuildKeyList($MaxNbOf{'LoginShown'},$MinHit{'Login'},\%_login_h,\%_login_p); foreach my $key (@keylist) { my $bredde_p=0;my $bredde_h=0;my $bredde_k=0; if ($max_h > 0) { $bredde_p=int($BarWidth*$_login_p{$key}/$max_h)+1; } # use max_h to enable to compare pages with hits if ($max_h > 0) { $bredde_h=int($BarWidth*$_login_h{$key}/$max_h)+1; } if ($max_k > 0) { $bredde_k=int($BarWidth*$_login_k{$key}/$max_k)+1; } print "$key"; &ShowUserInfo($key); if ($ShowAuthenticatedUsers =~ /P/i) { print "".($_login_p{$key}?$_login_p{$key}:" ").""; } if ($ShowAuthenticatedUsers =~ /H/i) { print "$_login_h{$key}"; } if ($ShowAuthenticatedUsers =~ /B/i) { print "".Format_Bytes($_login_k{$key}).""; } if ($ShowAuthenticatedUsers =~ /L/i) { print "".($_login_l{$key}?Format_Date($_login_l{$key},1):'-').""; } print "\n"; $total_p += $_login_p{$key}; $total_h += $_login_h{$key}; $total_k += $_login_k{$key}; $count++; } $rest_p=$TotalPages-$total_p; $rest_h=$TotalHits-$total_h; $rest_k=$TotalBytes-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other logins print "".($PageDir eq 'rtl'?"":"")."$Message[125]".($PageDir eq 'rtl'?"":"").""; &ShowUserInfo(''); if ($ShowAuthenticatedUsers =~ /P/i) { print "".($rest_p?$rest_p:" ").""; } if ($ShowAuthenticatedUsers =~ /H/i) { print "$rest_h"; } if ($ShowAuthenticatedUsers =~ /B/i) { print "".Format_Bytes($rest_k).""; } if ($ShowAuthenticatedUsers =~ /L/i) { print " "; } print "\n"; } &tab_end(); } # BY ROBOTS #---------------------------- if ($ShowRobotsStats) { if ($Debug) { debug("ShowRobotStats",2); } print "$Center 
\n"; &tab_head("$Message[53] ($Message[77] $MaxNbOf{'RobotShown'})   -   $Message[80]   -   $Message[9]",19,0,'robots'); print "".(scalar keys %_robot_h)." $Message[51]*"; if ($ShowRobotsStats =~ /H/i) { print "$Message[57]"; } if ($ShowRobotsStats =~ /B/i) { print "$Message[75]"; } if ($ShowRobotsStats =~ /L/i) { print "$Message[9]"; } print "\n"; $total_p=$total_h=$total_k=$total_r=0; my $count=0; &BuildKeyList($MaxNbOf{'RobotShown'},$MinHit{'Robot'},\%_robot_h,\%_robot_h); foreach my $key (@keylist) { print "".($PageDir eq 'rtl'?"":"").($RobotsHashIDLib{$key}?$RobotsHashIDLib{$key}:$key).($PageDir eq 'rtl'?"":"").""; if ($ShowRobotsStats =~ /H/i) { print "".($_robot_h{$key}-$_robot_r{$key}).($_robot_r{$key}?"+$_robot_r{$key}":"").""; } if ($ShowRobotsStats =~ /B/i) { print "".Format_Bytes($_robot_k{$key}).""; } if ($ShowRobotsStats =~ /L/i) { print "".($_robot_l{$key}?Format_Date($_robot_l{$key},1):'-').""; } print "\n"; #$total_p += $_robot_p{$key}; $total_h += $_robot_h{$key}; $total_k += $_robot_k{$key}||0; $total_r += $_robot_r{$key}||0; $count++; } # For bots we need to count Totals my $TotalPagesRobots = 0; #foreach (values %_robot_p) { $TotalPagesRobots+=$_; } my $TotalHitsRobots = 0; foreach (values %_robot_h) { $TotalHitsRobots+=$_; } my $TotalBytesRobots = 0; foreach (values %_robot_k) { $TotalBytesRobots+=$_; } my $TotalRRobots = 0; foreach (values %_robot_r) { $TotalRRobots+=$_; } $rest_p=0; #$rest_p=$TotalPagesRobots-$total_p; $rest_h=$TotalHitsRobots-$total_h; $rest_k=$TotalBytesRobots-$total_k; $rest_r=$TotalRRobots-$total_r; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0 || $rest_r > 0) { # All other robots print "$Message[2]"; if ($ShowRobotsStats =~ /H/i) { print "".($rest_h-$rest_r).($rest_r?"+$rest_r":"").""; } if ($ShowRobotsStats =~ /B/i) { print "".(Format_Bytes($rest_k)).""; } if ($ShowRobotsStats =~ /L/i) { print " "; } print "\n"; } &tab_end("* $Message[156]".($TotalRRobots?" $Message[157]":"")); } # BY WORMS #---------------------------- if ($ShowWormsStats) { if ($Debug) { debug("ShowWormsStats",2); } print "$Center 
\n"; &tab_head("$Message[163] ($Message[77] $MaxNbOf{'WormsShown'})",19,0,'worms'); print ""; print "".(scalar keys %_worm_h)." $Message[164]*"; print "$Message[167]"; if ($ShowWormsStats =~ /H/i) { print "$Message[57]"; } if ($ShowWormsStats =~ /B/i) { print "$Message[75]"; } if ($ShowWormsStats =~ /L/i) { print "$Message[9]"; } print "\n"; $total_p=$total_h=$total_k=0; my $count=0; &BuildKeyList($MaxNbOf{'WormsShown'},$MinHit{'Worm'},\%_worm_h,\%_worm_h); foreach my $key (@keylist) { print ""; print "".($PageDir eq 'rtl'?"":"").($WormsHashLib{$key}?$WormsHashLib{$key}:$key).($PageDir eq 'rtl'?"":"").""; print "".($PageDir eq 'rtl'?"":"").($WormsHashTarget{$key}?$WormsHashTarget{$key}:$key).($PageDir eq 'rtl'?"":"").""; if ($ShowWormsStats =~ /H/i) { print "".$_worm_h{$key}.""; } if ($ShowWormsStats =~ /B/i) { print "".Format_Bytes($_worm_k{$key}).""; } if ($ShowWormsStats =~ /L/i) { print "".($_worm_l{$key}?Format_Date($_worm_l{$key},1):'-').""; } print "\n"; #$total_p += $_worm_p{$key}; $total_h += $_worm_h{$key}; $total_k += $_worm_k{$key}||0; $count++; } # For worms we need to count Totals my $TotalPagesWorms = 0; #foreach (values %_worm_p) { $TotalPagesWorms+=$_; } my $TotalHitsWorms = 0; foreach (values %_worm_h) { $TotalHitsWorms+=$_; } my $TotalBytesWorms = 0; foreach (values %_worm_k) { $TotalBytesWorms+=$_; } $rest_p=0; #$rest_p=$TotalPagesRobots-$total_p; $rest_h=$TotalHitsWorms-$total_h; $rest_k=$TotalBytesWorms-$total_k; if ($rest_p > 0 || $rest_h > 0 || $rest_k > 0) { # All other worms print ""; print "$Message[2]"; print "-"; if ($ShowWormsStats =~ /H/i) { print "".($rest_h).""; } if ($ShowWormsStats =~ /B/i) { print "".(Format_Bytes($rest_k)).""; } if ($ShowWormsStats =~ /L/i) { print " "; } print "\n"; } &tab_end("* $Message[158]"); } print "\n \n\n"; # BY SESSION #---------------------------- if ($ShowSessionsStats) { if ($Debug) { debug("ShowSessionsStats",2); } print "$Center 
\n"; my $title="$Message[117]"; &tab_head($title,19,0,'sessions'); my $Totals=0; foreach (@SessionsRange) { $average_s+=($_session{$_}||0)*$SessionsAverage{$_}; $Totals+=$_session{$_}||0; } if ($Totals) { $average_s=int($average_s/$Totals); } else { $average_s='?'; } print "$Message[10]: $TotalVisits - $Message[96]: $average_s s$Message[10]$Message[15]\n"; $average_s=0; $total_s=0; my $count=0; foreach my $key (@SessionsRange) { my $p=0; if ($TotalVisits) { $p=int($_session{$key}/$TotalVisits*1000)/10; } $total_s+=$_session{$key}||0; print "$key"; print "".($_session{$key}?$_session{$key}:" ").""; print "".($_session{$key}?"$p %":" ").""; print "\n"; $count++; } $rest_s=$TotalVisits-$total_s; if ($rest_s > 0) { # All others sessions my $p=0; if ($TotalVisits) { $p=int($rest_s/$TotalVisits*1000)/10; } print "$Message[0]"; print "$rest_s"; print "".($rest_s?"$p %":" ").""; print "\n"; } &tab_end(); } # BY FILE TYPE #------------------------- if ($ShowFileTypesStats) { if ($Debug) { debug("ShowFileTypesStatsCompressionStats",2); } print "$Center 
\n"; my $Totalh=0; foreach (keys %_filetypes_h) { $Totalh+=$_filetypes_h{$_}; } my $Totalk=0; foreach (keys %_filetypes_k) { $Totalk+=$_filetypes_k{$_}; } my $title="$Message[73]"; if ($ShowFileTypesStats =~ /C/i) { $title.=" - $Message[98]"; } &tab_head("$title",19,0,'filetypes'); print "$Message[73]"; if ($ShowFileTypesStats =~ /H/i) { print "$Message[57]$Message[15]"; } if ($ShowFileTypesStats =~ /B/i) { print "$Message[75]$Message[15]"; } if ($ShowFileTypesStats =~ /C/i) { print "$Message[100]$Message[101]$Message[99]"; } print "\n"; my $total_con=0; my $total_cre=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,1,\%_filetypes_h,\%_filetypes_h); foreach my $key (@keylist) { my $p_h=' '; my $p_k=' '; if ($Totalh) { $p_h=int($_filetypes_h{$key}/$Totalh*1000)/10; $p_h="$p_h %"; } if ($Totalk) { $p_k=int($_filetypes_k{$key}/$Totalk*1000)/10; $p_k="$p_k %"; } if ($key eq 'Unknown') { print "$Message[0]"; } else { my $nameicon=$MimeHashIcon{$key}||"notavailable"; my $nametype=$MimeHashLib{$MimeHashFamily{$key}||""}||" "; print "$key"; print "$nametype"; } if ($ShowFileTypesStats =~ /H/i) { print "$_filetypes_h{$key}$p_h"; } if ($ShowFileTypesStats =~ /B/i) { print "".Format_Bytes($_filetypes_k{$key})."$p_k"; } if ($ShowFileTypesStats =~ /C/i) { if ($_filetypes_gz_in{$key}) { my $percent=int(100*(1-$_filetypes_gz_out{$key}/$_filetypes_gz_in{$key})); printf("%s%s%s (%s%)",Format_Bytes($_filetypes_gz_in{$key}),Format_Bytes($_filetypes_gz_out{$key}),Format_Bytes($_filetypes_gz_in{$key}-$_filetypes_gz_out{$key}),$percent); $total_con+=$_filetypes_gz_in{$key}; $total_cre+=$_filetypes_gz_out{$key}; } else { print "   "; } } print "\n"; $count++; } # Add total (only usefull if compression is enabled) if ($ShowFileTypesStats =~ /C/i) { my $colspan=3; if ($ShowFileTypesStats =~ /H/i) { $colspan+=2; } if ($ShowFileTypesStats =~ /B/i) { $colspan+=2; } print ""; print "$Message[98]"; if ($ShowFileTypesStats =~ /C/i) { if ($total_con) { my $percent=int(100*(1-$total_cre/$total_con)); printf("%s%s%s (%s%)",Format_Bytes($total_con),Format_Bytes($total_cre),Format_Bytes($total_con-$total_cre),$percent); } else { print "   "; } } print "\n"; } &tab_end(); } # BY FILE SIZE #------------------------- if ($ShowFileSizesStats) { } # BY FILE/URL #------------------------- if ($ShowPagesStats) { if ($Debug) { debug("ShowPagesStats (MaxNbOf{'PageShown'}=$MaxNbOf{'PageShown'} TotalDifferentPages=$TotalDifferentPages)",2); } print "$Center   
\n"; my $title="$Message[19] ($Message[77] $MaxNbOf{'PageShown'})   -   $Message[80]"; if ($ShowPagesStats =~ /E/i) { $title.="   -   $Message[104]"; } if ($ShowPagesStats =~ /X/i) { $title.="   -   $Message[116]"; } &tab_head("$title",19,0,'urls'); print "$TotalDifferentPages $Message[28]"; if ($ShowPagesStats =~ /P/i) { print "$Message[29]"; } if ($ShowPagesStats =~ /B/i) { print "$Message[106]"; } if ($ShowPagesStats =~ /E/i) { print "$Message[104]"; } if ($ShowPagesStats =~ /X/i) { print "$Message[116]"; } # Call to plugins' function ShowPagesAddField foreach my $pluginname (keys %{$PluginsLoaded{'ShowPagesAddField'}}) { my $function="ShowPagesAddField_$pluginname('title')"; eval("$function"); } print " \n"; $total_p=$total_e=$total_x=$total_k=0; $max_p=1; $max_k=1; my $count=0; &BuildKeyList($MaxNbOf{'PageShown'},$MinHit{'File'},\%_url_p,\%_url_p); foreach my $key (@keylist) { if ($_url_p{$key} > $max_p) { $max_p = $_url_p{$key}; } if ($_url_k{$key}/($_url_p{$key}||1) > $max_k) { $max_k = $_url_k{$key}/($_url_p{$key}||1); } } foreach my $key (@keylist) { print ""; &ShowURLInfo($key); print ""; my $bredde_p=0; my $bredde_e=0; my $bredde_x=0; my $bredde_k=0; if ($max_p > 0) { $bredde_p=int($BarWidth*($_url_p{$key}||0)/$max_p)+1; } if (($bredde_p==1) && $_url_p{$key}) { $bredde_p=2; } if ($max_p > 0) { $bredde_e=int($BarWidth*($_url_e{$key}||0)/$max_p)+1; } if (($bredde_e==1) && $_url_e{$key}) { $bredde_e=2; } if ($max_p > 0) { $bredde_x=int($BarWidth*($_url_x{$key}||0)/$max_p)+1; } if (($bredde_x==1) && $_url_x{$key}) { $bredde_x=2; } if ($max_k > 0) { $bredde_k=int($BarWidth*(($_url_k{$key}||0)/($_url_p{$key}||1))/$max_k)+1; } if (($bredde_k==1) && $_url_k{$key}) { $bredde_k=2; } if ($ShowPagesStats =~ /P/i) { print "$_url_p{$key}"; } if ($ShowPagesStats =~ /B/i) { print "".($_url_k{$key}?Format_Bytes($_url_k{$key}/($_url_p{$key}||1)):" ").""; } if ($ShowPagesStats =~ /E/i) { print "".($_url_e{$key}?$_url_e{$key}:" ").""; } if ($ShowPagesStats =~ /X/i) { print "".($_url_x{$key}?$_url_x{$key}:" ").""; } # Call to plugins' function ShowPagesAddField foreach my $pluginname (keys %{$PluginsLoaded{'ShowPagesAddField'}}) { my $function="ShowPagesAddField_$pluginname('$key')"; eval("$function"); } print ""; if ($ShowPagesStats =~ /P/i) { print "
"; } if ($ShowPagesStats =~ /B/i) { print "
"; } if ($ShowPagesStats =~ /E/i) { print "
"; } if ($ShowPagesStats =~ /X/i) { print ""; } print "\n"; $total_p += $_url_p{$key}; $total_e += $_url_e{$key}; $total_x += $_url_x{$key}; $total_k += $_url_k{$key}; $count++; } $rest_p=$TotalPages-$total_p; $rest_e=$TotalEntries-$total_e; $rest_x=$TotalExits-$total_x; $rest_k=$TotalBytesPages-$total_k; if ($rest_p > 0 || $rest_k > 0 || $rest_e > 0 || $rest_x > 0) { # All other urls print "$Message[2]"; if ($ShowPagesStats =~ /P/i) { print "$rest_p"; } if ($ShowPagesStats =~ /B/i) { print "".($rest_k?Format_Bytes($rest_k/($rest_p||1)):" ").""; } if ($ShowPagesStats =~ /E/i) { print "".($rest_e?$rest_e:" ").""; } if ($ShowPagesStats =~ /X/i) { print "".($rest_x?$rest_x:" ").""; } # Call to plugins' function ShowPagesAddField foreach my $pluginname (keys %{$PluginsLoaded{'ShowPagesAddField'}}) { my $function="ShowPagesAddField_$pluginname('')"; eval("$function"); } print " \n"; } &tab_end(); } # BY OS #---------------------------- if ($ShowOSStats) { if ($Debug) { debug("ShowOSStats",2); } print "$Center 
\n"; my $Totalh=0; my %new_os_h=(); OSLOOP: foreach my $key (keys %_os_h) { $Totalh+=$_os_h{$key}; foreach my $family (@OSFamily) { if ($key =~ /^$family/i) { $new_os_h{"${family}cumul"}+=$_os_h{$key}; next OSLOOP; } } $new_os_h{$key}+=$_os_h{$key}; } my $title="$Message[59] ($Message[77] $MaxNbOf{'OsShown'})   -   $Message[80]/$Message[58]   -   $Message[0]"; &tab_head("$title",19,0,'os'); print " $Message[59]$Message[57]$Message[15]\n"; $total_h=0; my $count=0; &BuildKeyList($MaxNbOf{'OsShown'},$MinHit{'Os'},\%new_os_h,\%new_os_h); foreach my $key (@keylist) { my $p=' '; if ($Totalh) { $p=int($new_os_h{$key}/$Totalh*1000)/10; $p="$p %"; } if ($key eq 'Unknown') { print "$Message[0]$_os_h{$key}$p\n"; } else { my $keywithoutcumul=$key; $keywithoutcumul =~ s/cumul$//i; my $libos=$OSHashLib{$keywithoutcumul}||$keywithoutcumul; my $nameicon=$keywithoutcumul; $nameicon =~ s/[^\w]//g; # TODO Use OSFamilyLib if ($libos eq 'win') { $libos="Windows"; } if ($libos eq 'mac') { $libos="Macintosh"; } print "$libos$new_os_h{$key}$p\n"; } $total_h += $new_os_h{$key}; $count++; } if ($Debug) { debug("Total real / shown : $Totalh / $total_h",2); } $rest_h=$Totalh-$total_h; if ($rest_h > 0) { my $p; if ($Totalh) { $p=int($rest_h/$Totalh*1000)/10; } print ""; print " "; print "$Message[2]$rest_h"; print "$p %\n"; } &tab_end(); } # BY BROWSER #---------------------------- if ($ShowBrowsersStats) { if ($Debug) { debug("ShowBrowsersStats",2); } print "$Center 
\n"; my $Totalh=0; my %new_browser_h=(); BROWSERLOOP: foreach my $key (keys %_browser_h) { $Totalh+=$_browser_h{$key}; foreach my $family (keys %BrowsersFamily) { if ($key =~ /^$family/i) { $new_browser_h{"${family}cumul"}+=$_browser_h{$key}; next BROWSERLOOP; } } $new_browser_h{$key}+=$_browser_h{$key}; } my $title="$Message[21] ($Message[77] $MaxNbOf{'BrowsersShown'})   -   $Message[80]/$Message[58]   -   $Message[0]"; &tab_head("$title",19,0,'browsers'); print " $Message[21]$Message[111]$Message[57]$Message[15]\n"; $total_h=0; my $count=0; &BuildKeyList($MaxNbOf{'BrowsersShown'},$MinHit{'Browser'},\%new_browser_h,\%new_browser_h); foreach my $key (@keylist) { my $p=' '; if ($Totalh) { $p=int($new_browser_h{$key}/$Totalh*1000)/10; $p="$p %"; } if ($key eq 'Unknown') { print "$Message[0]?$_browser_h{$key}$p\n"; } else { my $keywithoutcumul=$key; $keywithoutcumul =~ s/cumul$//i; my $libbrowser=$BrowsersHashIDLib{$keywithoutcumul}||$keywithoutcumul; my $nameicon=$BrowsersHashIcon{$keywithoutcumul}||"notavailable"; if ($BrowsersFamily{$keywithoutcumul}) { $libbrowser="$libbrowser"; } print "".($PageDir eq 'rtl'?"":"")."$libbrowser".($PageDir eq 'rtl'?"":"")."".($BrowsersHereAreGrabbers{$key}?"$Message[112]":"$Message[113]")."$new_browser_h{$key}$p\n"; } $total_h += $new_browser_h{$key}; $count++; } if ($Debug) { debug("Total real / shown : $Totalh / $total_h",2); } $rest_h=$Totalh-$total_h; if ($rest_h > 0) { my $p; if ($Totalh) { $p=int($rest_h/$Totalh*1000)/10; } print ""; print " "; print "$Message[2] $rest_h"; print "$p %\n"; } &tab_end(); } # BY SCREEN SIZE #---------------------------- if ($ShowScreenSizeStats) { if ($Debug) { debug("ShowScreenSizeStats",2); } print "$Center 
\n"; my $Totalh=0; foreach (keys %_screensize_h) { $Totalh+=$_screensize_h{$_}; } my $title="$Message[135] ($Message[77] $MaxNbOf{'ScreenSizesShown'})"; &tab_head("$title",0,0,'screensizes'); print "$Message[135]$Message[15]\n"; my $total_h=0; my $count=0; &BuildKeyList($MaxNbOf{'ScreenSizesShown'},$MinHit{'ScreenSize'},\%_screensize_h,\%_screensize_h); foreach my $key (@keylist) { my $p=' '; if ($Totalh) { $p=int($_screensize_h{$key}/$Totalh*1000)/10; $p="$p %"; } $total_h+=$_screensize_h{$key}||0; print ""; if ($key eq 'Unknown') { print "$Message[0]"; print "$p"; } else { my $screensize=$key; print "$screensize"; print "$p"; } print "\n"; $count++; } $rest_h=$Totalh-$total_h; if ($rest_h > 0) { # All others sessions my $p=0; if ($Totalh) { $p=int($rest_h/$Totalh*1000)/10; } print "$Message[2]"; print "".($rest_h?"$p %":" ").""; print "\n"; } &tab_end(); } print "\n \n\n"; # BY REFERENCE #--------------------------- if ($ShowOriginStats) { if ($Debug) { debug("ShowOriginStats",2); } print "$Center 
\n"; my $Totalp=0; foreach (0..5) { $Totalp+=($_ != 4 || $IncludeInternalLinksInOriginSection)?$_from_p[$_]:0; } my $Totalh=0; foreach (0..5) { $Totalh+=($_ != 4 || $IncludeInternalLinksInOriginSection)?$_from_h[$_]:0; } &tab_head($Message[36],19,0,'referer'); my @p_p=(0,0,0,0,0,0); if ($Totalp > 0) { $p_p[0]=int($_from_p[0]/$Totalp*1000)/10; $p_p[1]=int($_from_p[1]/$Totalp*1000)/10; $p_p[2]=int($_from_p[2]/$Totalp*1000)/10; $p_p[3]=int($_from_p[3]/$Totalp*1000)/10; $p_p[4]=int($_from_p[4]/$Totalp*1000)/10; $p_p[5]=int($_from_p[5]/$Totalp*1000)/10; } my @p_h=(0,0,0,0,0,0); if ($Totalh > 0) { $p_h[0]=int($_from_h[0]/$Totalh*1000)/10; $p_h[1]=int($_from_h[1]/$Totalh*1000)/10; $p_h[2]=int($_from_h[2]/$Totalh*1000)/10; $p_h[3]=int($_from_h[3]/$Totalh*1000)/10; $p_h[4]=int($_from_h[4]/$Totalh*1000)/10; $p_h[5]=int($_from_h[5]/$Totalh*1000)/10; } print "$Message[37]"; if ($ShowOriginStats =~ /P/i) { print "$Message[56]$Message[15]"; } if ($ShowOriginStats =~ /H/i) { print "$Message[57]$Message[15]"; } print "\n"; #------- Referrals by direct address/bookmarks print "$Message[38]"; if ($ShowOriginStats =~ /P/i) { print "".($_from_p[0]?$_from_p[0]:" ")."".($_from_p[0]?"$p_p[0] %":" ").""; } if ($ShowOriginStats =~ /H/i) { print "".($_from_h[0]?$_from_h[0]:" ")."".($_from_h[0]?"$p_h[0] %":" ").""; } print "\n"; #------- Referrals by news group print "$Message[107]"; if ($ShowOriginStats =~ /P/i) { print "".($_from_p[5]?$_from_p[5]:" ")."".($_from_p[5]?"$p_p[5] %":" ").""; } if ($ShowOriginStats =~ /H/i) { print "".($_from_h[5]?$_from_h[5]:" ")."".($_from_h[5]?"$p_h[5] %":" ").""; } print "\n"; #------- Referrals by search engines print "$Message[40] - $Message[80]
\n"; if (scalar keys %_se_referrals_h) { print "\n"; $total_p=0; $total_h=0; my $count=0; &BuildKeyList($MaxNbOf{'RefererShown'},$MinHit{'Refer'},\%_se_referrals_h,((scalar keys %_se_referrals_p)?\%_se_referrals_p:\%_se_referrals_h)); foreach my $key (@keylist) { my $newreferer=CleanFromCSSA($SearchEnginesHashLib{$key}||$key); print ""; print ""; print ""; print "\n"; $total_p += $_se_referrals_p{$key}; $total_h += $_se_referrals_h{$key}; $count++; } if ($Debug) { debug("Total real / shown : $TotalSearchEnginesPages / $total_p - $TotalSearchEnginesHits / $total_h",2); } $rest_p=$TotalSearchEnginesPages-$total_p; $rest_h=$TotalSearchEnginesHits-$total_h; if ($rest_p > 0 || $rest_h > 0) { print ""; print ""; print ""; print "\n"; } print "
- $newreferer".($_se_referrals_p{$key}?$_se_referrals_p{$key}:'0')."$_se_referrals_h{$key}
- $Message[2]$rest_p$rest_h
"; } print "\n"; if ($ShowOriginStats =~ /P/i) { print "".($_from_p[2]?$_from_p[2]:" ")."".($_from_p[2]?"$p_p[2] %":" ").""; } if ($ShowOriginStats =~ /H/i) { print "".($_from_h[2]?$_from_h[2]:" ")."".($_from_h[2]?"$p_h[2] %":" ").""; } print "\n"; #------- Referrals by external HTML link print "$Message[41] - $Message[80]
\n"; if (scalar keys %_pagesrefs_h) { print "\n"; $total_p=0; $total_h=0; my $count=0; &BuildKeyList($MaxNbOf{'RefererShown'},$MinHit{'Refer'},\%_pagesrefs_h,((scalar keys %_pagesrefs_p)?\%_pagesrefs_p:\%_pagesrefs_h)); foreach my $key (@keylist) { print ""; print ""; print ""; print "\n"; $total_p += $_pagesrefs_p{$key}; $total_h += $_pagesrefs_h{$key}; $count++; } if ($Debug) { debug("Total real / shown : $TotalRefererPages / $total_p - $TotalRefererHits / $total_h",2); } $rest_p=$TotalRefererPages-$total_p; $rest_h=$TotalRefererHits-$total_h; if ($rest_p > 0 || $rest_h > 0) { print ""; print ""; print ""; print "\n"; } print "
- "; &ShowURLInfo($key); print "".($_pagesrefs_p{$key}?$_pagesrefs_p{$key}:'0')."$_pagesrefs_h{$key}
- $Message[2]$rest_p$rest_h
"; } print "\n"; if ($ShowOriginStats =~ /P/i) { print "".($_from_p[3]?$_from_p[3]:" ")."".($_from_p[3]?"$p_p[3] %":" ").""; } if ($ShowOriginStats =~ /H/i) { print "".($_from_h[3]?$_from_h[3]:" ")."".($_from_h[3]?"$p_h[3] %":" ").""; } print "\n"; #------- Referrals by internal HTML link if ($IncludeInternalLinksInOriginSection) { print "$Message[42]"; if ($ShowOriginStats =~ /P/i) { print "".($_from_p[4]?$_from_p[4]:" ")."".($_from_p[4]?"$p_p[4] %":" ").""; } if ($ShowOriginStats =~ /H/i) { print "".($_from_h[4]?$_from_h[4]:" ")."".($_from_h[4]?"$p_h[4] %":" ").""; } print "\n"; } #------- Unknown origin print "$Message[39]"; if ($ShowOriginStats =~ /P/i) { print "".($_from_p[1]?$_from_p[1]:" ")."".($_from_p[1]?"$p_p[1] %":" ").""; } if ($ShowOriginStats =~ /H/i) { print "".($_from_h[1]?$_from_h[1]:" ")."".($_from_h[1]?"$p_h[1] %":" ").""; } print "\n"; &tab_end(); } print "\n \n\n"; # BY SEARCH KEYWORDS AND/OR KEYPHRASES #------------------------------------- if ($ShowKeyphrasesStats) { print "$Center "; } if ($ShowKeywordsStats) { print "$Center "; } if ($ShowKeyphrasesStats || $ShowKeywordsStats) { print "
\n"; } if ($ShowKeyphrasesStats && $ShowKeywordsStats) { print ""; } if ($ShowKeyphrasesStats) { # By Keyphrases if ($ShowKeyphrasesStats && $ShowKeywordsStats) { print "\n"; $total_s=0; my $count=0; &BuildKeyList($MaxNbOf{'KeyphrasesShown'},$MinHit{'Keyphrase'},\%_keyphrases,\%_keyphrases); foreach my $key (@keylist) { my $mot; # Convert coded keywords (utf8,...) to be correctly reported in HTML page. if ($PluginsLoaded{'DecodeKey'}{'decodeutfkeys'}) { $mot=CleanFromCSSA(DecodeKey_decodeutfkeys($key,$PageCode||'iso-8859-1')); } else { $mot = CleanFromCSSA(DecodeEncodedString($key)); } my $p; if ($TotalKeyphrases) { $p=int($_keyphrases{$key}/$TotalKeyphrases*1000)/10; } print "\n"; $total_s += $_keyphrases{$key}; $count++; } if ($Debug) { debug("Total real / shown : $TotalKeyphrases / $total_s",2); } $rest_s=$TotalKeyphrases-$total_s; if ($rest_s > 0) { my $p; if ($TotalKeyphrases) { $p=int($rest_s/$TotalKeyphrases*1000)/10; } print ""; print "\n"; } &tab_end(); if ($ShowKeyphrasesStats && $ShowKeywordsStats) { print "\n"; } } if ($ShowKeyphrasesStats && $ShowKeywordsStats) { print ""; } if ($ShowKeywordsStats) { # By Keywords if ($ShowKeyphrasesStats && $ShowKeywordsStats) { print "\n"; $total_s=0; my $count=0; &BuildKeyList($MaxNbOf{'KeywordsShown'},$MinHit{'Keyword'},\%_keywords,\%_keywords); foreach my $key (@keylist) { my $mot; # Convert coded keywords (utf8,...) to be correctly reported in HTML page. if ($PluginsLoaded{'DecodeKey'}{'decodeutfkeys'}) { $mot=CleanFromCSSA(DecodeKey_decodeutfkeys($key,$PageCode||'iso-8859-1')); } else { $mot = CleanFromCSSA(DecodeEncodedString($key)); } my $p; if ($TotalKeywords) { $p=int($_keywords{$key}/$TotalKeywords*1000)/10; } print "\n"; $total_s += $_keywords{$key}; $count++; } if ($Debug) { debug("Total real / shown : $TotalKeywords / $total_s",2); } $rest_s=$TotalKeywords-$total_s; if ($rest_s > 0) { my $p; if ($TotalKeywords) { $p=int($rest_s/$TotalKeywords*1000)/10; } print ""; print "\n"; } &tab_end(); if ($ShowKeyphrasesStats && $ShowKeywordsStats) { print "\n"; } } if ($ShowKeyphrasesStats && $ShowKeywordsStats) { print "
\n"; } if ($Debug) { debug("ShowKeyphrasesStats",2); } &tab_head("$Message[120] ($Message[77] $MaxNbOf{'KeyphrasesShown'})
$Message[80]",19,($ShowKeyphrasesStats && $ShowKeywordsStats)?95:70,'keyphrases'); print "
$TotalDifferentKeyphrases $Message[103]$Message[14]$Message[15]
".XMLEncode($mot)."$_keyphrases{$key}$p %
$Message[124]$rest_s$p %
  \n"; } if ($Debug) { debug("ShowKeywordsStats",2); } &tab_head("$Message[121] ($Message[77] $MaxNbOf{'KeywordsShown'})
$Message[80]",19,($ShowKeyphrasesStats && $ShowKeywordsStats)?95:70,'keywords'); print "
$TotalDifferentKeywords $Message[13]$Message[14]$Message[15]
".XMLEncode($mot)."$_keywords{$key}$p %
$Message[30]$rest_s$p %
\n"; } print "\n \n\n"; # BY MISC #---------------------------- if ($ShowMiscStats) { if ($Debug) { debug("ShowMiscStats",2); } print "$Center 
\n"; my $Totalh=0; my %new_browser_h=(); if ($_misc_h{'AddToFavourites'}) { foreach my $key (keys %_browser_h) { $Totalh+=$_browser_h{$key}; if ($key =~ /^msie/i) { $new_browser_h{"msiecumul"}+=$_browser_h{$key}; } } if ($new_browser_h{'msiecumul'}) { $_misc_h{'AddToFavourites'}=int(0.5+$_misc_h{'AddToFavourites'}*$Totalh/$new_browser_h{'msiecumul'}); } } my $title="$Message[139]"; &tab_head("$title",19,0,'misc'); print "$Message[139]"; print " "; print " "; print "\n"; my %label=('AddToFavourites'=>$Message[137],'JavascriptDisabled'=>$Message[168],'JavaEnabled'=>$Message[140],'DirectorSupport'=>$Message[141], 'FlashSupport'=>$Message[142],'RealPlayerSupport'=>$Message[143],'QuickTimeSupport'=>$Message[144], 'WindowsMediaPlayerSupport'=>$Message[145],'PDFSupport'=>$Message[146]); foreach my $key (@MiscListOrder) { my $mischar=substr($key,0,1); if ($ShowMiscStats !~ /$mischar/i) { next; } my $total=0; my $p; if ($MiscListCalc{$key} eq 'v') { $total=$TotalVisits; } if ($MiscListCalc{$key} eq 'u') { $total=$TotalUnique; } if ($MiscListCalc{$key} eq 'hm') { $total=$_misc_h{'TotalMisc'}||0; } if ($total) { $p=int($_misc_h{$key}/$total*1000)/10; } print ""; print "".($PageDir eq 'rtl'?"":"").$label{$key}.($PageDir eq 'rtl'?"":"").""; if ($MiscListCalc{$key} eq 'v') { print "".($_misc_h{$key}||0)." / $total $Message[12]"; } if ($MiscListCalc{$key} eq 'u') { print "".($_misc_h{$key}||0)." / $total $Message[18]"; } if ($MiscListCalc{$key} eq 'hm') { print "-"; } print "".($total?"$p %":" ").""; print "\n"; } &tab_end(); } # BY HTTP STATUS #---------------------------- if ($ShowHTTPErrorsStats) { if ($Debug) { debug("ShowHTTPErrorsStats",2); } print "$Center 
\n"; my $title="$Message[32]"; &tab_head("$title",19,0,'errors'); print "$Message[32]*$Message[57]$Message[15]$Message[75]\n"; $total_h=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,1,\%_errors_h,\%_errors_h); foreach my $key (@keylist) { my $p=int($_errors_h{$key}/$TotalHitsErrors*1000)/10; print ""; if ($TrapInfosForHTTPErrorCodes{$key}) { print "$key"; } else { print "$key"; } print "".($httpcodelib{$key}?$httpcodelib{$key}:'Unknown error')."$_errors_h{$key}$p %".Format_Bytes($_errors_k{$key}).""; print "\n"; $total_h+=$_errors_h{$key}; $count++; } &tab_end("* $Message[154]"); } # BY SMTP STATUS #---------------------------- if ($ShowSMTPErrorsStats) { if ($Debug) { debug("ShowSMTPErrorsStats",2); } print "$Center 
\n"; my $title="$Message[147]"; &tab_head("$title",19,0,'errors'); print "$Message[147]$Message[57]$Message[15]$Message[75]\n"; $total_h=0; my $count=0; &BuildKeyList($MaxRowsInHTMLOutput,1,\%_errors_h,\%_errors_h); foreach my $key (@keylist) { my $p=int($_errors_h{$key}/$TotalHitsErrors*1000)/10; print ""; print "$key"; print "".($smtpcodelib{$key}?$smtpcodelib{$key}:'Unknown error')."$_errors_h{$key}$p %".Format_Bytes($_errors_k{$key}).""; print "\n"; $total_h+=$_errors_h{$key}; $count++; } &tab_end(); } # BY CLUSTER #---------------------------- if ($ShowClusterStats) { if ($Debug) { debug("ShowClusterStats",2); } print "$Center 
\n"; my $title="$Message[155]"; &tab_head("$title",19,0,'clusters'); print "$Message[155]"; &ShowClusterInfo('__title__'); if ($ShowClusterStats =~ /P/i) { print "$Message[56]$Message[15]"; } if ($ShowClusterStats =~ /H/i) { print "$Message[57]$Message[15]"; } if ($ShowClusterStats =~ /B/i) { print "$Message[75]$Message[15]"; } print "\n"; $total_p=$total_h=$total_k=0; # Cluster feature might have been enable in middle of month so we recalculate # total for cluster section only, to calculate ratio, instead of using global total foreach my $key (keys %_cluster_h) { $total_p+=int($_cluster_p{$key}||0); $total_h+=int($_cluster_h{$key}||0); $total_k+=int($_cluster_k{$key}||0); } my $count=0; foreach my $key (keys %_cluster_h) { my $p_p=int($_cluster_p{$key}/$total_p*1000)/10; my $p_h=int($_cluster_h{$key}/$total_h*1000)/10; my $p_k=int($_cluster_k{$key}/$total_k*1000)/10; print ""; print "Computer $key"; &ShowClusterInfo($key); if ($ShowClusterStats =~ /P/i) { print "".($_cluster_p{$key}?$_cluster_p{$key}:" ")."$p_p %"; } if ($ShowClusterStats =~ /H/i) { print "$_cluster_h{$key}$p_h %"; } if ($ShowClusterStats =~ /B/i) { print "".Format_Bytes($_cluster_k{$key})."$p_k %"; } print "\n"; $count++; } &tab_end(); } # BY EXTRA SECTIONS #---------------------------- foreach my $extranum (1..@ExtraName-1) { if ($Debug) { debug("ExtraName$extranum",2); } print "$Center 
"; my $title=$ExtraName[$extranum]; &tab_head("$title",19,0,"extra$extranum"); print ""; print "".$ExtraFirstColumnTitle[$extranum].""; if ($ExtraStatTypes[$extranum] =~ m/P/i) { print "$Message[56]"; } if ($ExtraStatTypes[$extranum] =~ m/H/i) { print "$Message[57]"; } if ($ExtraStatTypes[$extranum] =~ m/B/i) { print "$Message[75]"; } if ($ExtraStatTypes[$extranum] =~ m/L/i) { print "$Message[9]"; } print "\n"; $total_p=$total_h=$total_k=0; #$max_h=1; foreach (values %_login_h) { if ($_ > $max_h) { $max_h = $_; } } #$max_k=1; foreach (values %_login_k) { if ($_ > $max_k) { $max_k = $_; } } my $count=0; if ($ExtraStatTypes[$extranum] =~ m/P/i) { &BuildKeyList($MaxNbOfExtra[$extranum],$MinHitExtra[$extranum],\%{'_section_' . $extranum . '_h'},\%{'_section_' . $extranum . '_p'}); } else { &BuildKeyList($MaxNbOfExtra[$extranum],$MinHitExtra[$extranum],\%{'_section_' . $extranum . '_h'},\%{'_section_' . $extranum . '_h'}); } foreach my $key (@keylist) { my $firstcol = CleanFromCSSA(DecodeEncodedString($key)); $total_p+=${'_section_' . $extranum . '_p'}{$key}; $total_h+=${'_section_' . $extranum . '_h'}{$key}; $total_k+=${'_section_' . $extranum . '_k'}{$key}; print ""; printf("$ExtraFirstColumnFormat[$extranum]", $firstcol, $firstcol, $firstcol, $firstcol, $firstcol); if ($ExtraStatTypes[$extranum] =~ m/P/i) { print "" . ${'_section_' . $extranum . '_p'}{$key} . ""; } if ($ExtraStatTypes[$extranum] =~ m/H/i) { print "" . ${'_section_' . $extranum . '_h'}{$key} . ""; } if ($ExtraStatTypes[$extranum] =~ m/B/i) { print "" . Format_Bytes(${'_section_' . $extranum . '_k'}{$key}) . ""; } if ($ExtraStatTypes[$extranum] =~ m/L/i) { print "" . (${'_section_' . $extranum . '_l'}{$key}?Format_Date(${'_section_' . $extranum . '_l'}{$key},1):'-') . ""; } print "\n"; $count++; } if ($ExtraAddAverageRow[$extranum]) { print ""; print "$Message[96]"; if ($ExtraStatTypes[$extranum] =~ m/P/i) { print "" . ($count?($total_p/$count):" ") . ""; } if ($ExtraStatTypes[$extranum] =~ m/H/i) { print "" . ($count?($total_h/$count):" ") . ""; } if ($ExtraStatTypes[$extranum] =~ m/B/i) { print "" . ($count?Format_Bytes($total_k/$count):" ") . ""; } if ($ExtraStatTypes[$extranum] =~ m/L/i) { print " "; } print "\n"; } if ($ExtraAddSumRow[$extranum]) { print ""; print "$Message[102]"; if ($ExtraStatTypes[$extranum] =~ m/P/i) { print "" . ($total_p) . ""; } if ($ExtraStatTypes[$extranum] =~ m/H/i) { print "" . ($total_h) . ""; } if ($ExtraStatTypes[$extranum] =~ m/B/i) { print "" . Format_Bytes($total_k) . ""; } if ($ExtraStatTypes[$extranum] =~ m/L/i) { print " "; } print "\n"; } &tab_end(); } &html_end(1); } } else { print "Jumped lines in file: $lastlinenb\n"; if ($lastlinenb) { print " Found $lastlinenb already parsed records.\n"; } print "Parsed lines in file: $NbOfLinesParsed\n"; print " Found $NbOfLinesDropped dropped records,\n"; print " Found $NbOfLinesCorrupted corrupted records,\n"; print " Found $NbOfOldLines old records,\n"; print " Found $NbOfNewLines new qualified records.\n"; } #sleep 10; 0; # Do not remove this line #------------------------------------------------------- # ALGORITHM SUMMARY # # Read_Config(); # Check_Config() and Init variables # if 'frame not index' # &Read_Language_Data($Lang); # if 'frame not mainleft' # &Read_Ref_Data(); # &Read_Plugins(); # html_head # # If 'migrate' # We create/update tmp file with # &Read_History_With_TmpUpdate(year,month,UPDATE,NOPURGE,"all"); # Rename the tmp file # html_end # Exit # End of 'migrate' # # Get last history file name # Get value for $LastLine $LastLineNumber $LastLineOffset $LastLineChecksum with # &Read_History_With_TmpUpdate(lastyear,lastmonth,NOUPDATE,NOPURGE,"general"); # # &Init_HashArray() # # If 'update' # Loop on each new line in log file # lastlineoffset=lastlineoffsetnext; lastlineoffsetnext=file pointer position # If line corrupted, skip --> next on loop # Drop wrong virtual host --> next on loop # Drop wrong method/protocol --> next on loop # Check date --> next on loop # If line older than $LastLine, skip --> next on loop # So it's new line # $LastLine = time or record # Skip if url is /robots.txt --> next on loop # Skip line for @SkipHosts --> next on loop # Skip line for @SkipFiles --> next on loop # Skip line for @SkipUserAgent --> next on loop # Skip line for not @OnlyHosts --> next on loop # Skip line for not @OnlyFiles --> next on loop # Skip line for not @OnlyUserAgent --> next on loop # Skip line for not @OnlyUsers --> next on loop # Skip line for not @OnlyLines --> next on loop # So it's new line approved # If other month/year, create/update tmp file and purge data arrays with # &Read_History_With_TmpUpdate(lastprocessedyear,lastprocessedmonth,UPDATE,PURGE,"all",lastlinenb,lastlineoffset,CheckSum($_)); # Define a clean Url and Query (set urlwithnoquery, tokenquery and standalonequery and $field[$pos_url]) # Define PageBool and extension # Analyze: Misc tracker --> complete %misc # Analyze: Add to favorites --> complete %_misc, countedtraffic=1 (not counted anywhere) # If (!countedtraffic) Analyze: Worms --> complete %_worms, countedtraffic=1 # If (!countedtraffic) Analyze: Status code --> complete %_error_, %_sider404, %_referrer404 --> countedtraffic=1 # If (!countedtraffic) Analyze: Robots known --> complete %_robot, countedtraffic=1 # If (!countedtraffic) Analyze: Robots unknown on robots.txt --> complete %_robot, countedtraffic=1 # If (!countedtraffic) Analyze: File types - Compression # If (!countedtraffic) Analyze: Date - Hour - Pages - Hits - Kilo # If (!countedtraffic) Analyze: Login # If (!countedtraffic) Do DNS Lookup # If (!countedtraffic) Analyze: Country # If (!countedtraffic) Analyze: Host - Url - Session # If (!countedtraffic) Analyze: Browser - OS # If (!countedtraffic) Analyze: Referer # If (!countedtraffic) Analyze: EMail # Analyze: Cluster # Analyze: Extra (must be after 'Define a clean Url and Query') # If too many records, we flush data arrays with # &Read_History_With_TmpUpdate($lastprocessedyear,$lastprocessedmonth,UPDATE,PURGE,"all",lastlinenb,lastlineoffset,CheckSum($_)); # End of loop # Create/update tmp file # Seek to lastlineoffset to read and get last line into $_ # &Read_History_With_TmpUpdate($lastprocessedyear,$lastprocessedmonth,UPDATE,PURGE,"all",lastlinenb,lastlineoffset,CheckSum($_)) # Rename all tmp files # End of 'update' # # &Init_HashArray() # # If 'output' # Loop for each month of required year # &Read_History_With_TmpUpdate($YearRequired,monthloop,NOUPDATE,NOPURGE,"all" or "general time" if not required month) # End of loop # Show data arrays in HTML page # html_end # End of 'output' #------------------------------------------------------- #------------------------------------------------------- # DNS CACHE FILE FORMATS SUPPORTED BY AWSTATS # Format /etc/hosts x.y.z.w hostname # Format analog UT/60 x.y.z.w hostname #------------------------------------------------------- #------------------------------------------------------- # IP Format (d=decimal on 16 bits, x=hexadecimal on 16 bits) # # 13.1.68.3 IPv4 (d.d.d.d) # 0:0:0:0:0:0:13.1.68.3 IPv6 (x:x:x:x:x:x:d.d.d.d) # ::13.1.68.3 # 0:0:0:0:0:FFFF:13.1.68.3 IPv6 (x:x:x:x:x:x:d.d.d.d) # ::FFFF:13.1.68.3 IPv6 # # 1070:0:0:0:0:800:200C:417B IPv6 # 1070:0:0:0:0:800:200C:417B IPv6 # 1070::800:200C:417B IPv6 #-------------------------------------------------------