YUKI Hiroshi
null+****@clear*****
Wed Dec 10 17:40:51 JST 2014
YUKI Hiroshi 2014-12-10 17:40:51 +0900 (Wed, 10 Dec 2014) New Revision: 875f887e7f2e87de577f288d786200d0c58ea80f https://github.com/droonga/log-search/commit/875f887e7f2e87de577f288d786200d0c58ea80f Message: Add installation script for Ubuntu Added files: ubuntu/install.sh ubuntu/log-parsing-node/td-agent.conf ubuntu/log-search-node/td-agent.conf ubuntu/monitoring-target/td-agent.conf Added: ubuntu/install.sh (+125 -0) 100755 =================================================================== --- /dev/null +++ ubuntu/install.sh 2014-12-10 17:40:51 +0900 (f706600) @@ -0,0 +1,125 @@ +: ${ROLE:=stand-alone} + +BASE_URL=https://raw.githubusercontent.com/droonga/log-search/master/ubuntu +GROONGA_ADMIN_VERSION=0.9.1 + +case $(uname) in + Darwin|*BSD|CYGWIN*) sed="sed -E" ;; + *) sed="sed -r" ;; +esac + +ensure_root() { + if [ "$EUID" != "0" ]; then + echo "You must run this script as the root." + exit 1 + fi +} + +exist_command() { + type "$1" > /dev/null 2>&1 +} + +setup_as_log_search_node() { + if ! exist_command groonga + then + echo "Installing Groonga..." + add-apt-repository -y ppa:groonga/ppa + apt-get update + apt-get install -y groonga + fi + + if [ ! -d /usr/share/groonga/html/groonga-admin ] + then + echo "Installing Groonga admin page..." + cd /tmp + wget http://packages.groonga.org/source/groonga-admin/groonga-admin-$GROONGA_ADMIN_VERSION.tar.gz + tar xvf groonga-admin-$GROONGA_ADMIN_VERSION.tar.gz + sudo cp -r groonga-admin-$GROONGA_ADMIN_VERSION/html /usr/share/groonga/html/groonga-admin + fi + + # mkdir ~/groonga-log-search + # groonga -n ~/groonga-log-search/db quit + # groonga -d \ + # -p 10041 \ + # --protocol http \ + # --document-root /usr/share/groonga/html/groonga-admin \ + # ~/groonga-log-search/db + + echo "Installing Fluentd plugins for a log-search-node..." + td-agent-gem install fluent-plugin-secure-forward + td-agent-gem install fluent-plugin-groonga + + if [ "$ROLE" = "log-search-node" ] + then + curl -o /etc/td-agent/td-agent.conf "$BASE_URL/log-search-node/td-agent.conf" + fi +} + +setup_as_log_parsing_node() { + echo "Installing Fluentd plugins for a log-parsing-node..." + td-agent-gem install fluent-plugin-secure-forward + td-agent-gem install fluent-plugin-forest + td-agent-gem install fluent-plugin-parser + td-agent-gem install fluent-plugin-record-reformer + + if [ "$ROLE" = "log-parsing-node" ] + then + curl -o /etc/td-agent/td-agent.conf "$BASE_URL/log-parsing-node/td-agent.conf" + fi +} + +setup_as_monitoring_target() { + echo "Installing Fluentd plugins for a monitoring-target..." + td-agent-gem install fluent-plugin-secure-forward + td-agent-gem install fluent-plugin-config-expander + + echo "Updating permission of the system log file for monitoring..." + chmod g+r /var/log/syslog + chgrp td-agent /var/log/syslog + + if [ "$ROLE" = "monitoring-target" ] + then + curl -o /etc/td-agent/td-agent.conf "$BASE_URL/monitoring-target/td-agent.conf" + fi +} + +install() { + echo "Installing and activating npd..." + apt-get install ntp + service ntp restart + + echo "Installing Fluentd..." + curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sh + + echo "Preparing the buffer directory..." + mkdir -p /var/spool/td-agent/buffer/ + chown -R td-agent:td-agent /var/spool/td-agent/ + + if [ "$ROLE" = "log-search-node" -o "$ROLE" = "stand-alone" ] + then + setup_as_log_search_node + fi + + if [ "$ROLE" = "log-parsing-node" -o "$ROLE" = "stand-alone" ] + then + setup_as_log_parsing_node + fi + + if [ "$ROLE" = "monitoring-target" -o "$ROLE" = "stand-alone" ] + then + setup_as_monitoring_target + fi + + if [ "$ROLE" = "stand-alone" ] + then + curl -o /etc/td-agent/td-agent.conf "$BASE_URL/stand-alone/td-agent.conf" + fi + + service td-agent restart +} + +ensure_root + +install + +exit 0 Added: ubuntu/log-parsing-node/td-agent.conf (+47 -0) 100644 =================================================================== --- /dev/null +++ ubuntu/log-parsing-node/td-agent.conf 2014-12-10 17:40:51 +0900 (3de779a) @@ -0,0 +1,47 @@ +<source> + type secure_forward + shared_key fluentd-secret + self_hostname parser1.example.com + cert_auto_generate yes +</source> + +<match raw.*.log.**> + type forest + subtype parser + + <template> + key_name message + </template> + + <case raw.messages.log.**> + remove_prefix raw + format syslog + </case> +</match> + +<match *.log.*.**> + type record_reformer + enable_ruby false + + tag ${tag_parts[1]} + + <record> + host ${tag_suffix[2]} + type ${tag_parts[0]} + timestamp ${time} + </record> +</match> + +<match log> + type secure_forward + shared_key fluentd-secret + self_hostname parser1.example.com + + buffer_type file + buffer_path /var/spool/td-agent/buffer/secure-forward + flush_interval 1 + + <server> + host search.example.com + </server> +</match> Added: ubuntu/log-search-node/td-agent.conf (+66 -0) 100644 =================================================================== --- /dev/null +++ ubuntu/log-search-node/td-agent.conf 2014-12-10 17:40:51 +0900 (0ad0622) @@ -0,0 +1,66 @@ +<source> + type secure_forward + shared_key fluentd-secret + self_hostname search.example.com + cert_auto_generate yes +</source> + +<match log> + type groonga + store_table Logs + + protocol http + host 127.0.0.1 + + buffer_type file + buffer_path /var/spool/td-agent/buffer/groonga + flush_interval 1 + + <table> + name Terms + flags TABLE_PAT_KEY + key_type ShortText + default_tokenizer TokenBigram + normalizer NormalizerAuto + </table> + + <table> + name Hosts + flags TABLE_PAT_KEY + key_type ShortText + # normalizer NormalizerAuto + </table> + + <table> + name Timestamps + flags TABLE_PAT_KEY + key_type Time + </table> + + <mapping> + name host + type Hosts + <index> + table Hosts + name logs_index + </index> + </mapping> + + <mapping> + name timestamp + type Time + <index> + table Timestamps + name logs_index + </index> + </mapping> + + <mapping> + name message + type Text + <index> + table Terms + name logs_message_index + </index> + </mapping> +</match> Added: ubuntu/monitoring-target/td-agent.conf (+26 -0) 100644 =================================================================== --- /dev/null +++ ubuntu/monitoring-target/td-agent.conf 2014-12-10 17:40:51 +0900 (07bbb87) @@ -0,0 +1,26 @@ +<source> + type config_expander + <config> + type tail + path /var/log/syslog + pos_file /var/log/td-agent/messages.pos + tag raw.messages.log.${hostname} + format none + </config> +</source> + +<match raw.*.log.**> + type secure_forward + shared_key fluentd-secret + self_hostname node1.example.com + <server> + host parser1.example.com + </server> + <server> + host parser2.example.com + </server> + + buffer_type file + buffer_path /var/spool/td-agent/buffer/secure-forward + flush_interval 1 +</match> -------------- next part -------------- HTML����������������������������...Download
