Kouhei Sutou
null+****@clear*****
Tue Nov 7 14:34:15 JST 2017
Kouhei Sutou 2017-11-07 14:34:15 +0900 (Tue, 07 Nov 2017) New Revision: 48193e5ad24ace1b00fa8d67c89fe2cc9e5be464 https://github.com/groonga/groonga/commit/48193e5ad24ace1b00fa8d67c89fe2cc9e5be464 Message: logical_count: support dynamic column * "initial" stage is only supported because "filtered" stage and "output" stage are meaningless. * Add "reason" to "use range index" log. (It should be done in separated commit. :p) TODO: * Document Added files: test/command/suite/sharding/logical_count/cache/columns/initial.expected test/command/suite/sharding/logical_count/cache/columns/initial.test test/command/suite/sharding/logical_count/columns/stage/initial/filter.expected test/command/suite/sharding/logical_count/columns/stage/initial/filter.test Modified files: plugins/sharding/logical_count.rb test/command/suite/sharding/logical_count/log/cover_type_all.expected test/command/suite/sharding/logical_count/log/range_index.expected test/command/suite/sharding/logical_count/log/select.expected Modified: plugins/sharding/logical_count.rb (+139 -106) =================================================================== --- plugins/sharding/logical_count.rb 2017-11-07 14:33:06 +0900 (8bdd77ef7) +++ plugins/sharding/logical_count.rb 2017-11-07 14:34:15 +0900 (de0039c20) @@ -14,12 +14,11 @@ module Groonga def run_body(input) enumerator = LogicalEnumerator.new("logical_count", input) - filter = input[:filter] + counter = Counter.new(input, enumerator.target_range) total = 0 enumerator.each do |shard, shard_range| - total += count_n_records(filter, shard, shard_range, - enumerator.target_range) + total += counter.count(shard, shard_range) end writer.write(total) end @@ -34,133 +33,167 @@ module Groonga key << "#{input[:max]}\0" key << "#{input[:max_border]}\0" key << "#{input[:filter]}\0" + dynamic_columns = DynamicColumns.parse(input) + key << dynamic_columns.cache_key key end - def log_use_range_index(use, table_name, line, method) - message = "[logical_count]" - if use - message << "[range-index]" - else - message << "[select]" + class Counter + def initialize(input, target_range) + @logger = Context.instance.logger + @filter = input[:filter] + @dynamic_columns = DynamicColumns.parse(input) + @target_range = target_range end - message << " <#{table_name}>" - Context.instance.logger.log(Logger::Level::DEBUG, - __FILE__, - line, - method.to_s, - message) - end - def count_n_records(filter, shard, shard_range, target_range) - cover_type = target_range.cover_type(shard_range) - return 0 if cover_type == :none + def count(shard, shard_range) + cover_type = @target_range.cover_type(shard_range) + return 0 if cover_type == :none - shard_key = shard.key - if shard_key.nil? - message = "[logical_count] shard_key doesn't exist: " + - "<#{shard.key_name}>" - raise InvalidArgument, message - end - table = shard.table - table_name = shard.table_name - - expression_builder = RangeExpressionBuilder.new(shard_key, - target_range) - expression_builder.filter = filter - if cover_type == :all - log_use_range_index(false, table_name, __LINE__, __method__) - if filter.nil? - return table.size - else - return filtered_count_n_records(table) do |expression| - expression_builder.build_all(expression) + shard_key = shard.key + if shard_key.nil? + message = "[logical_count] shard_key doesn't exist: " + + "<#{shard.key_name}>" + raise InvalidArgument, message + end + table_name = shard.table_name + + prepare_table(shard) do |table| + if cover_type == :all + log_use_range_index(false, table_name, "covered", + __LINE__, __method__) + if @filter + return filtered_count_n_records(table, shard_key, cover_type) + else + return table.size + end + end + + range_index = nil + if @filter + log_use_range_index(false, table_name, "need filter", + __LINE__, __method__) + else + index_info = shard_key.find_index(Operator::LESS) + range_index = index_info.index if index_info + if range_index + log_use_range_index(true, table_name, "range index is available", + __LINE__, __method__) + else + log_use_range_index(false, table_name, "no range index", + __LINE__, __method__) + end + end + + if range_index + count_n_records_in_range(range_index, cover_type) + else + filtered_count_n_records(table, shard_key, cover_type) end end end - range_index = nil - if filter.nil? - index_info = shard_key.find_index(Operator::LESS) - if index_info - range_index = index_info.index + private + def log_use_range_index(use, table_name, reason, line, method) + message = "[logical_count]" + if use + message << "[range-index]" + else + message << "[select]" end + message << " <#{table_name}>: #{reason}" + @logger.log(Logger::Level::DEBUG, + __FILE__, + line, + method.to_s, + message) end - use_range_index = (!range_index.nil?) - log_use_range_index(use_range_index, table_name, __LINE__, __method__) + def prepare_table(shard) + table = shard.table + return yield(table) if****@filte*****? - case cover_type - when :partial_min - if range_index - count_n_records_in_range(range_index, - target_range.min, target_range.min_border, - nil, nil) - else - filtered_count_n_records(table) do |expression| - expression_builder.build_partial_min(expression) + @dynamic_columns.each_initial do |dynamic_column| + if table == shard.table + table = table.select_all end + dynamic_column.apply(table) end - when :partial_max - if range_index - count_n_records_in_range(range_index, - nil, nil, - target_range.max, target_range.max_border) - else - filtered_count_n_records(table) do |expression| - expression_builder.build_partial_max(expression) - end + + begin + yield(table) + ensure + table.close if table != shard.table end - when :partial_min_and_max - if range_index - count_n_records_in_range(range_index, - target_range.min, target_range.min_border, - target_range.max, target_range.max_border) - else - filtered_count_n_records(table) do |expression| + end + + def filtered_count_n_records(table, shard_key, cover_type) + expression = nil + filtered_table = nil + + expression_builder = RangeExpressionBuilder.new(shard_key, + @target_range) + expression_builder.filter = @filter + begin + expression = Expression.create(table) + case cover_type + when :all + expression_builder.build_all(expression) + when :partial_min + expression_builder.build_partial_min(expression) + when :partial_max + expression_builder.build_partial_max(expression) + when :partial_min_and_max expression_builder.build_partial_min_and_max(expression) end + filtered_table = table.select(expression) + filtered_table.size + ensure + filtered_table.close if filtered_table + expression.close if expression end end - end - def filtered_count_n_records(table) - expression = nil - filtered_table = nil - - begin - expression = Expression.create(table) - yield(expression) - filtered_table = table.select(expression) - filtered_table.size - ensure - filtered_table.close if filtered_table - expression.close if expression - end - end + def count_n_records_in_range(range_index, cover_type) + case cover_type + when :partial_min + min = @target_range.min + min_border = @target_range.min_border + max = nil + max_bordre = nil + when :partial_max + min = nil + min_bordre = nil + max = @target_range.max + max_border = @target_range.max_border + when :partial_min_and_max + min = @target_range.min + min_border = @target_range.min_border + max = @target_range.max + max_border = @target_range.max_border + end - def count_n_records_in_range(range_index, - min, min_border, max, max_border) - flags = TableCursorFlags::BY_KEY - case min_border - when :include - flags |= TableCursorFlags::GE - when :exclude - flags |= TableCursorFlags::GT - end - case max_border - when :include - flags |= TableCursorFlags::LE - when :exclude - flags |= TableCursorFlags::LT - end + flags = TableCursorFlags::BY_KEY + case min_border + when :include + flags |= TableCursorFlags::GE + when :exclude + flags |= TableCursorFlags::GT + end + case max_border + when :include + flags |= TableCursorFlags::LE + when :exclude + flags |= TableCursorFlags::LT + end - TableCursor.open(range_index.table, - :min => min, - :max => max, - :flags => flags) do |table_cursor| - IndexCursor.open(table_cursor, range_index) do |index_cursor| - index_cursor.count + TableCursor.open(range_index.table, + :min => min, + :max => max, + :flags => flags) do |table_cursor| + IndexCursor.open(table_cursor, range_index) do |index_cursor| + index_cursor.count + end end end end Added: test/command/suite/sharding/logical_count/cache/columns/initial.expected (+26 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_count/cache/columns/initial.expected 2017-11-07 14:34:15 +0900 (482bfd226) @@ -0,0 +1,26 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +logical_count Logs --shard_key timestamp --columns[price_with_tax].stage initial --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --filter 'price_with_tax > 550' +[[0,0.0,0.0],2] +#>logical_count --columns[price_with_tax].flags "COLUMN_SCALAR" --columns[price_with_tax].stage "initial" --columns[price_with_tax].type "UInt32" --columns[price_with_tax].value "price * 1.08" --filter "price_with_tax > 550" --logical_table "Logs" --shard_key "timestamp" +#:000000000000000 filter(3) +#:000000000000000 filter(2) +#<000000000000000 rc=0 +logical_count Logs --shard_key timestamp --columns[price_with_tax].stage initial --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --filter 'price_with_tax > 550' +[[0,0.0,0.0],2] +#>logical_count --columns[price_with_tax].flags "COLUMN_SCALAR" --columns[price_with_tax].stage "initial" --columns[price_with_tax].type "UInt32" --columns[price_with_tax].value "price * 1.08" --filter "price_with_tax > 550" --logical_table "Logs" --shard_key "timestamp" +#:000000000000000 cache(0) +#<000000000000000 rc=0 Added: test/command/suite/sharding/logical_count/cache/columns/initial.test (+33 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_count/cache/columns/initial.test 2017-11-07 14:34:15 +0900 (cfc11407b) @@ -0,0 +1,33 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +#@sleep 1 + +#@collect-query-log true +logical_count Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage initial \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --filter 'price_with_tax > 550' +logical_count Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage initial \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --filter 'price_with_tax > 550' +#@collect-query-log false Added: test/command/suite/sharding/logical_count/columns/stage/initial/filter.expected (+58 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_count/columns/stage/initial/filter.expected 2017-11-07 14:34:15 +0900 (636add034) @@ -0,0 +1,58 @@ +plugin_register sharding +[[0,0.0,0.0],true] +table_create Logs_20170315 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170315 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170316 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170316 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +table_create Logs_20170317 TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +[[0,0.0,0.0],true] +column_create Logs_20170317 price COLUMN_SCALAR UInt32 +[[0,0.0,0.0],true] +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] +[[0,0.0,0.0],3] +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] +[[0,0.0,0.0],3] +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] +[[0,0.0,0.0],3] +table_create Times TABLE_PAT_KEY Time +[[0,0.0,0.0],true] +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +[[0,0.0,0.0],true] +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp +[[0,0.0,0.0],true] +log_level --level debug +[[0,0.0,0.0],true] +logical_count Logs --shard_key timestamp --columns[price_with_tax].stage initial --columns[price_with_tax].type UInt32 --columns[price_with_tax].flags COLUMN_SCALAR --columns[price_with_tax].value 'price * 1.08' --filter 'price_with_tax > 550' +[[0,0.0,0.0],5] +#|d| [logical_count][select] <Logs_20170315>: covered +#|d| [logical_count][select] <Logs_20170316>: covered +#|d| [logical_count][select] <Logs_20170317>: covered +log_level --level notice +[[0,0.0,0.0],true] Added: test/command/suite/sharding/logical_count/columns/stage/initial/filter.test (+53 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/sharding/logical_count/columns/stage/initial/filter.test 2017-11-07 14:34:15 +0900 (bc5efed1f) @@ -0,0 +1,53 @@ +#@on-error omit +plugin_register sharding +#@on-error default + +table_create Logs_20170315 TABLE_NO_KEY +column_create Logs_20170315 timestamp COLUMN_SCALAR Time +column_create Logs_20170315 price COLUMN_SCALAR UInt32 + +table_create Logs_20170316 TABLE_NO_KEY +column_create Logs_20170316 timestamp COLUMN_SCALAR Time +column_create Logs_20170316 price COLUMN_SCALAR UInt32 + +table_create Logs_20170317 TABLE_NO_KEY +column_create Logs_20170317 timestamp COLUMN_SCALAR Time +column_create Logs_20170317 price COLUMN_SCALAR UInt32 + +load --table Logs_20170315 +[ +{"timestamp": "2017/03/15 00:00:00", "price": 1000}, +{"timestamp": "2017/03/15 01:00:00", "price": 900}, +{"timestamp": "2017/03/15 02:00:00", "price": 300} +] + +load --table Logs_20170316 +[ +{"timestamp": "2017/03/16 10:00:00", "price": 530}, +{"timestamp": "2017/03/16 11:00:00", "price": 520}, +{"timestamp": "2017/03/16 12:00:00", "price": 110} +] + +load --table Logs_20170317 +[ +{"timestamp": "2017/03/17 20:00:00", "price": 800}, +{"timestamp": "2017/03/17 21:00:00", "price": 400}, +{"timestamp": "2017/03/17 22:00:00", "price": 300} +] + +table_create Times TABLE_PAT_KEY Time +column_create Times logs_20170315 COLUMN_INDEX Logs_20170315 timestamp +column_create Times logs_20170316 COLUMN_INDEX Logs_20170316 timestamp +column_create Times logs_20170317 COLUMN_INDEX Logs_20170317 timestamp + +#@add-important-log-levels debug +log_level --level debug +logical_count Logs \ + --shard_key timestamp \ + --columns[price_with_tax].stage initial \ + --columns[price_with_tax].type UInt32 \ + --columns[price_with_tax].flags COLUMN_SCALAR \ + --columns[price_with_tax].value 'price * 1.08' \ + --filter 'price_with_tax > 550' +log_level --level notice +#@remove-important-log-levels debug Modified: test/command/suite/sharding/logical_count/log/cover_type_all.expected (+1 -1) =================================================================== --- test/command/suite/sharding/logical_count/log/cover_type_all.expected 2017-11-07 14:33:06 +0900 (ec5dab88e) +++ test/command/suite/sharding/logical_count/log/cover_type_all.expected 2017-11-07 14:34:15 +0900 (cbd126a23) @@ -12,6 +12,6 @@ log_level --level debug [[0,0.0,0.0],true] logical_count Logs timestamp [[0,0.0,0.0],0] -#|d| [logical_count][select] <Logs_20150709> +#|d| [logical_count][select] <Logs_20150709>: covered log_level --level notice [[0,0.0,0.0],true] Modified: test/command/suite/sharding/logical_count/log/range_index.expected (+1 -1) =================================================================== --- test/command/suite/sharding/logical_count/log/range_index.expected 2017-11-07 14:33:06 +0900 (2ba1c1e51) +++ test/command/suite/sharding/logical_count/log/range_index.expected 2017-11-07 14:34:15 +0900 (1cefdda52) @@ -12,6 +12,6 @@ log_level --level debug [[0,0.0,0.0],true] logical_count Logs timestamp --min "2015/07/09 00:00:00" --max "2015/07/09 12:00:00" [[0,0.0,0.0],0] -#|d| [logical_count][range-index] <Logs_20150709> +#|d| [logical_count][range-index] <Logs_20150709>: range index is available log_level --level notice [[0,0.0,0.0],true] Modified: test/command/suite/sharding/logical_count/log/select.expected (+1 -1) =================================================================== --- test/command/suite/sharding/logical_count/log/select.expected 2017-11-07 14:33:06 +0900 (737d2f07b) +++ test/command/suite/sharding/logical_count/log/select.expected 2017-11-07 14:34:15 +0900 (fa5d02a76) @@ -8,6 +8,6 @@ log_level --level debug [[0,0.0,0.0],true] logical_count Logs timestamp --min "2015/07/09 00:00:00" --max "2015/07/09 12:00:00" [[0,0.0,0.0],0] -#|d| [logical_count][select] <Logs_20150709> +#|d| [logical_count][select] <Logs_20150709>: no range index log_level --level notice [[0,0.0,0.0],true] -------------- next part -------------- HTML����������������������������... URL: https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20171107/0ed1bdef/attachment-0001.htm
