PyukiWiki CVS Commit
pyuki****@lists*****
2012年 1月 30日 (月) 19:21:14 JST
Index: PyukiWiki-Devel/lib/Nana/Logs.pm diff -u PyukiWiki-Devel/lib/Nana/Logs.pm:1.99 PyukiWiki-Devel/lib/Nana/Logs.pm:1.100 --- PyukiWiki-Devel/lib/Nana/Logs.pm:1.99 Mon Jan 30 09:24:47 2012 +++ PyukiWiki-Devel/lib/Nana/Logs.pm Mon Jan 30 19:21:14 2012 @@ -1,8 +1,8 @@ ###################################################################### # Logs.pm - This is PyukiWiki, yet another Wiki clone. -# $Id: Logs.pm,v 1.99 2012/01/30 00:24:47 papu Exp $ +# $Id: Logs.pm,v 1.100 2012/01/30 10:21:14 papu Exp $ # -# "Nana::Logs" version 0.1 $$ +# "Nana::Logs" version 0.2 $$ # Author: Nanami # http://nanakochi.daiba.cx/ # Copyright (C) 2004-2012 Nekyo @@ -22,10 +22,11 @@ use 5.8.1; #use strict; use vars qw($VERSION); -$VERSION = '0.1'; +$VERSION = '0.2'; $LOGS::Load=0; -use Nana::YukiWikiDB; -use Nana::YukiWikiDB_GZIP; +#use Nana::YukiWikiDB; +#use Nana::YukiWikiDB_GZIP; +use Nana::Cache; my @SearchEnginesSearchIDOrder; my @RobotsSearchIDOrder; @@ -78,105 +79,221 @@ &init; - my %dates; - my %hours; - my %weeks; - - my %host; - my %hosts; - my %tmpcountries; -# my %countries; - my %topdomains; - my %domains; - my %tmptopdomains; - my %tmpdomains; - - my %agents; - my %uabrowser; - my %uabrowserver; - my %browsertype; - my %browserversion; - my %uaos; - my %os; + my %dates=(); + my %hours=(); + my %weeks=(); + + my %host=(); + my %hosts=(); + my %tmpcountries=(); +# my %countries=(); + my %topdomains=(); + my %domains=(); + my %tmptopdomains=(); + my %tmpdomains=(); + + my %agents=(); + my %uabrowser=(); + my %uabrowserver=(); + my %browsertype=(); + my %browserversion=(); + my %uaos=(); + my %os=(); - my %page; + my %page=(); my $pages=0; - my %write; + my %write=(); my $writes=0; - my %attachdownload; + my %attachdownload=(); my $attachdownloads=0; - my %attachpost; + my %attachpost=(); my $attachposts=0; - my %user; + my %user=(); my $users=0; - my %links; + my %links=(); - my %referer; - my %allreferer; - my %searchengine; - my %keywords; + my %referer=(); + my %allreferer=(); + my %searchengine=(); + my %keywords=(); my $counts=0; + + my %hash; foreach my $list(split(/,/,$lists)) { - foreach my $log(split(/\n/,$db{$list})) { - $counts++; - my($hosts,$dates,$user,$method,$cmd,$lang,$page,$agent,$refer) - = split(/\t/,$log); - my($host,$ip)=split(/ /,$hosts); - my($date,$week,$time)=split(/ /,$dates); - my($date_y, $date_m, $date_d)=split(/-/,$date); - my($time_h, $time_m, $time_s)=split(/:/,$time); - $dates{$date}++; - $hours{$time_h}++; - $weeks{$week}++; - - # ページのカウント - if($cmd eq "read") { - $page{"$lang\t$page"}++; - $pages++; - } + %hash=&analysis_sub($db{$list}, $list); + $counts+=$hash{count}; + $pages+=$hash{pagecount}; + $writes+=$hash{writes}; + $attachdownloads+=$hash{attachdownloads}; + $attachuploads+=$hash{attachuploads}; + + &addhash("dates", \%dates, \%hash); + &addhash("hours", \%hours, \%hash); + &addhash("weeks", \%weeks, \%hash); + &addhash("hosts", \%hosts, \%hash); +# &addhash("countries", \%countries, \%hash); + &addhash("topdomains", \%topdomains, \%hash); + &addhash("domains", \%domains, \%hash); + &addhash("uaos", \%os, \%hash); + &addhash("browsertypes", \%browsertype, \%hash); + &addhash("browserversions", \%browserversion, \%hash); + &addhash("pages", \%page, \%hash); + &addhash("links", \%links, \%hash); + &addhash("write", \%write, \%hash); + &addhash("attachdownload", \%attachdownload, \%hash); + &addhash("attachpost", \%attachpost, \%hash); + &addhash("users", \%user, \%hash); + &addhash("referers", \%referer, \%hash); + &addhash("agents", \%agents, \%hash); + &addhash("allreferers", \%allreferer, \%hash); + &addhash("searchengines", \%searchengine, \%hash); + &addhash("keywords", \%keywords, \%hash); - # ページ書き込みのカウント - if(($cmd eq "write" || $cmd=~/edit/ || $cmd=~/comment/ - || $cmd=~/article/ || $cmd eq "bugtrack" || $cmd eq "vote") - && $method eq "POST") { - $write{"$lang\t$page"}++; - $writes++; - } + } - # リンクのカウント - if($cmd eq "ck") { - $links{"$lang\t$page"}++; - } + return( + count => $counts, + pagecount => $pages, + writecount => $writes, + attachdownloads => $attachdownloads, + attachposts => $attachposts, + dates => \%dates, + hours => \%hours, + weeks => \%weeks, + hosts => \%hosts, +# countries => \%countries, + topdomains => \%topdomains, + domains => \%domains, + uaos => \%os, + browsertypes => \%browsertype, + browserversions => \%browserversion, + pages => \%page, + links => \%links, + write => \%write, + attachdownload => \%attachdownload, + attachpost => \%attachpost, + users => \%user, + referers => \%referer, + agents => \%agents, + allreferers => \%allreferer, + searchengines => \%searchengine, + keywords => \%keywords, + ); +} - # 添付ダウンロードのカウント - if($cmd eq "attach-open") { - $attachdownload{"$lang\t$page"}++; - $attachdownloads++; - } +sub addhash { + my ($k, $val, $hash)=@_; + my %h=%{$hash{$k}}; + my %v=%{$val}; + foreach (keys $hash->{$k}) { + $$val{$_}=$$hash{$k}->{$_}; + } +} - # 添付アップロードのカウント - if(($cmd eq "attach-post" || $cmd eq "attach-delete") - && $method eq "POST") { - $attachpost{"$lang\t$page"}++; - $attachposts++; - } +sub analysis_sub { + my ($data, $target)=@_; - # 添付アップロードのカウント - if($cmd eq "attach-post" || $cmd eq "attach-delete") { - $attachpost{"$lang\t$page"}++; - $attachposts++; - } + my %dates=(); + my %hours=(); + my %weeks=(); + + my %host=(); + my %hosts=(); + my %tmpcountries=(); +# my %countries=(); + my %topdomains=(); + my %domains=(); + my %tmptopdomains=(); + my %tmpdomains=(); + + my %agents=(); + my %uabrowser=(); + my %uabrowserver=(); + my %browsertype=(); + my %browserversion=(); + my %uaos=(); + my %os=(); - # ユーザーのカウント - $user{$user}++; + my %page=(); + my $pages=0; + my %write=(); + my $writes=0; + my %attachdownload=(); + my $attachdownloads=0; + my %attachpost=(); + my $attachposts=0; + my %user=(); + my $users=0; + + my %links=(); - # ホストのカウント - $hosts{"$hosts ($ip)"}++; - my $domain; + my %referer=(); + my %allreferer=(); + my %searchengine=(); + my %keywords=(); + + my $counts=0; + + foreach my $log(split(/\n/,$data)) { + $counts++; + my($hosts,$dates,$user,$method,$cmd,$lang,$page,$agent,$refer) + = split(/\t/,$log); + my($host,$ip)=split(/ /,$hosts); + my($date,$week,$time)=split(/ /,$dates); + my($date_y, $date_m, $date_d)=split(/-/,$date); + my($time_h, $time_m, $time_s)=split(/:/,$time); + $dates{$date}++; + $hours{$time_h}++; + $weeks{$week}++; + + # ページのカウント + if($cmd eq "read") { + $page{"$lang\t$page"}++; + $pages++; + } + + # ページ書き込みのカウント + if(($cmd eq "write" || $cmd=~/edit/ || $cmd=~/comment/ + || $cmd=~/article/ || $cmd eq "bugtrack" || $cmd eq "vote") + && $method eq "POST") { + $write{"$lang\t$page"}++; + $writes++; + } + + # リンクのカウント + if($cmd eq "ck") { + $links{"$lang\t$page"}++; + } + + # 添付ダウンロードのカウント + if($cmd eq "attach-open") { + $attachdownload{"$lang\t$page"}++; + $attachdownloads++; + } + + # 添付アップロードのカウント + if(($cmd eq "attach-post" || $cmd eq "attach-delete") + && $method eq "POST") { + $attachpost{"$lang\t$page"}++; + $attachposts++; + } + + # 添付アップロードのカウント + if($cmd eq "attach-post" || $cmd eq "attach-delete") { + $attachpost{"$lang\t$page"}++; + $attachposts++; + } + + # ユーザーのカウント + $user{$user}++; + + # ホストのカウント + $hosts{"$hosts ($ip)"}++; + my $domain; # my $tmpcountries=lc $host; - my $tmpdomain=lc $host; + my $tmpdomain=lc $host; # if($tmpcountries{$ip} eq '') { # foreach(keys %DomainsHashIDLib) { # my $top=$_; @@ -191,132 +308,131 @@ # } # } # } - if($tmptopdomains{$ip} eq '') { - foreach(keys %DomainsHashIDLib) { - my $top=$_; - my $regex=$_; - $regex=~s/\./\\\./g; - if($tmpdomain=~/\.$regex$/) { - $tmptopdomains{$ip}="$top ($DomainsHashIDLib{$top})"; - my $domain=$tmpdomain; - $domain=~s/\.$regex$//g; - $domain=~s/.*\.//g; - $tmpdomains{$ip}="$domain.$top"; - last; - } + if($tmptopdomains{$ip} eq '') { + foreach(keys %DomainsHashIDLib) { + my $top=$_; + my $regex=$_; + $regex=~s/\./\\\./g; + if($tmpdomain=~/\.$regex$/) { + $tmptopdomains{$ip}="$top ($DomainsHashIDLib{$top})"; + my $domain=$tmpdomain; + $domain=~s/\.$regex$//g; + $domain=~s/.*\.//g; + $tmpdomains{$ip}="$domain.$top"; + last; } - if($tmptopdomains{$ip} eq '') { - my $regipv4='^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' - + '|(::ffff:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'; - my $regipv4='^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}'; - my $regipv6='((([0-9a-f]{1,4}:){7}([0-9a-f]{1,4}|:))|(([0-9a-f]{1,4}:){6}(:[0-9a-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){5}(((:[0-9a-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){4}(((:[0-9a-f]{1,4}){1,3})|((:[0-9a-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){3}(((:[0-9a-f]{1,4}){1,4})|((:[0-9a-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){2}(((:[0-9a-f]{1,4}){1,5})|((:[0-9a-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){1}(((:[0-9a-f]{1,4}){1,6})|((:[0-9a-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9a-f]{1,4}){1,7})|((:[0-9a-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2 [0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?'; - if($ip=~/$regipv4/) { - $tmptopdomains{$ip}="IPV4 Address"; - } elsif($ip=~/$regipv6/) { - $tmptopdomains{$ip}="IPV6 Address"; - } else { - $tmptopdomains{$ip}=$host ? $host : $ip; - } - $tmpdomains{$ip}=$host ? $host : $ip; + } + if($tmptopdomains{$ip} eq '') { + my $regipv4='^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' + + '|(::ffff:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'; + my $regipv4='^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}'; + my $regipv6='((([0-9a-f]{1,4}:){7}([0-9a-f]{1,4}|:))|(([0-9a-f]{1,4}:){6}(:[0-9a-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){5}(((:[0-9a-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9a-f]{1,4}:){4}(((:[0-9a-f]{1,4}){1,3})|((:[0-9a-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){3}(((:[0-9a-f]{1,4}){1,4})|((:[0-9a-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){2}(((:[0-9a-f]{1,4}){1,5})|((:[0-9a-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9a-f]{1,4}:){1}(((:[0-9a-f]{1,4}){1,6})|((:[0-9a-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9a-f]{1,4}){1,7})|((:[0-9a-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[ 0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?'; + if($ip=~/$regipv4/) { + $tmptopdomains{$ip}="IPV4 Address"; + } elsif($ip=~/$regipv6/) { + $tmptopdomains{$ip}="IPV6 Address"; + } else { + $tmptopdomains{$ip}=$host ? $host : $ip; } + $tmpdomains{$ip}=$host ? $host : $ip; } - $topdomains{$tmptopdomains{$ip}}++; - $domains{$tmpdomains{$ip}}++; + } + $topdomains{$tmptopdomains{$ip}}++; + $domains{$tmpdomains{$ip}}++; - # ブラウザーのカウント - $agents{$agent}++; - my $browser=lc $agent; - - if(! $uaos{$browser}) { - foreach my $regex(@OSSearchIDOrder) { - if($browser=~/$regex/) { - $uaos{$browser}=&target($OSHashLib{$OSHashID{$regex}}); - last; - } + # ブラウザーのカウント + $agents{$agent}++; + my $browser=lc $agent; + + if(! $uaos{$browser}) { + foreach my $regex(@OSSearchIDOrder) { + if($browser=~/$regex/) { + $uaos{$browser}=&target($OSHashLib{$OSHashID{$regex}}); + last; } } - if(! $uaos{$browser}) { - $uaos{$browser}='Unknown'; - } - $os{$uaos{$browser}}++; + } + if(! $uaos{$browser}) { + $uaos{$browser}='Unknown'; + } + $os{$uaos{$browser}}++; - if(! $uabrowser{$browser}) { - my $found = 0; - if(!$found) { - foreach(@RobotsSearchIDOrder) { - if($browser =~ /$_/) { - $uabrowser{$browser}='Robot'; - $uabrowserver{$browser}=&target($RobotsHashIDLib{$_}); - $found=1; - last; - } + if(! $uabrowser{$browser}) { + my $found = 0; + if(!$found) { + foreach(@RobotsSearchIDOrder) { + if($browser =~ /$_/) { + $uabrowser{$browser}='Robot'; + $uabrowserver{$browser}=&target($RobotsHashIDLib{$_}); + $found=1; + last; } } - if(!$found) { - foreach my $id(@BrowsersFamily) { - if($browser=~/$BrowsersVersionHashIDLib{$id}/) { - my $version=$2 eq '' ? $1 : $2; - if($id eq "safari") { - $version= - $BrowsersSafariBuildToVersionHash{$version} - . " ($version)"; - } - $found=1; - $uabrowser{$browser}=$BrowsersHashIDLib{$id}; - $uabrowserver{$browser}="$BrowsersHashIDLib{$id}/$version"; - last; + } + if(!$found) { + foreach my $id(@BrowsersFamily) { + if($browser=~/$BrowsersVersionHashIDLib{$id}/) { + my $version=$2 eq '' ? $1 : $2; + if($id eq "safari") { + $version= + $BrowsersSafariBuildToVersionHash{$version} + . " ($version)"; } + $found=1; + $uabrowser{$browser}=$BrowsersHashIDLib{$id}; + $uabrowserver{$browser}="$BrowsersHashIDLib{$id}/$version"; + last; } } - if(!$found) { - foreach (@BrowsersSearchIDOrder) { - if($browser =~ /$_/ ) { - my $browserver = $browser; - $browserver=~s/.*$_[_+\/ ]([\d\.]*).*/$1/; - $uabrowser{$browser}=$BrowsersHashIDLib{$_}; - $uabrowserver{$browser}="$_/$browserver"; - $found=1; - last; - } + } + if(!$found) { + foreach (@BrowsersSearchIDOrder) { + if($browser =~ /$_/ ) { + my $browserver = $browser; + $browserver=~s/.*$_[_+\/ ]([\d\.]*).*/$1/; + $uabrowser{$browser}=$BrowsersHashIDLib{$_}; + $uabrowserver{$browser}="$_/$browserver"; + $found=1; + last; } } + } - if(!$found) { - $uabrowser{$browser}='Unknown'; - $uabrowserver{$browser}='Unknown'; - } + if(!$found) { + $uabrowser{$browser}='Unknown'; + $uabrowserver{$browser}='Unknown'; } + } - $browsertype{$uabrowser{$browser}}++; - $browserversion{$uabrowserver{$browser}}++; + $browsertype{$uabrowser{$browser}}++; + $browserversion{$uabrowserver{$browser}}++; - # リンク元のカウント - $refer=~s/&/&/g; - $found=0; - - foreach(@SearchEnginesSearchIDOrder) { - if($refer=~/$_/) { - $searchengine{$SearchEnginesHashLib{$SearchEnginesHashID{$_}}}++; - my $query=$SearchEnginesKnownUrl{$SearchEnginesHashID{$_}}; - my $q=$refer; - $q=~s/\?/&/g; - foreach $u(split(/&/,$q)) { - if($u=~/^$query/) { - my $tmp=&decode($u); - $tmp=~s/^$query//g; - my $word=&code_convert(\$tmp,$::defaultcode); - $keywords{"$page - $word"}++; - } + # リンク元のカウント + $refer=~s/&/&/g; + $found=0; + + foreach(@SearchEnginesSearchIDOrder) { + if($refer=~/$_/) { + $searchengine{$SearchEnginesHashLib{$SearchEnginesHashID{$_}}}++; + my $query=$SearchEnginesKnownUrl{$SearchEnginesHashID{$_}}; + my $q=$refer; + $q=~s/\?/&/g; + foreach $u(split(/&/,$q)) { + if($u=~/^$query/) { + my $tmp=&decode($u); + $tmp=~s/^$query//g; + my $word=&code_convert(\$tmp,$::defaultcode); + $keywords{"$page - $word"}++; } } } + } - if(!$found) { - $referer{$refer}++; - } - $allreferer{$refer}++; - + if(!$found) { + $referer{$refer}++; } + $allreferer{$refer}++; + } return( Index: PyukiWiki-Devel/lib/Nana/YukiWikiDB_GZIP.pm diff -u PyukiWiki-Devel/lib/Nana/YukiWikiDB_GZIP.pm:1.150 PyukiWiki-Devel/lib/Nana/YukiWikiDB_GZIP.pm:1.151 --- PyukiWiki-Devel/lib/Nana/YukiWikiDB_GZIP.pm:1.150 Mon Jan 30 16:41:15 2012 +++ PyukiWiki-Devel/lib/Nana/YukiWikiDB_GZIP.pm Mon Jan 30 19:21:14 2012 @@ -1,6 +1,6 @@ ###################################################################### # YukiWikiDB_GZIP.pm - This is PyukiWiki, yet another Wiki clone. -# $Id: YukiWikiDB_GZIP.pm,v 1.150 2012/01/30 07:41:15 papu Exp $ +# $Id: YukiWikiDB_GZIP.pm,v 1.151 2012/01/30 10:21:14 papu Exp $ # # "Nana::YukiWikiDB_GZIP" version 0.6 $$ # Author: Nanami @@ -125,7 +125,7 @@ $key=$2; } $key =~ s/(.)/$::_dbmname_encode{$1}/g; - return $self->{dir} . "/$key.txt"; + return ($mode, $self->{dir} . "/$key.txt"); } sub make_filename_gz { @@ -136,6 +136,6 @@ $key=$2; } $key =~ s/(.)/$::_dbmname_encode{$1}/g; - return $self->{dir} . "/$key.txt.gz"; + return ($mode, $self->{dir} . "/$key.txt"); } 1;
