naoa
null+****@clear*****
Wed Nov 5 12:35:44 JST 2014
naoa 2014-11-05 12:35:44 +0900 (Wed, 05 Nov 2014) New Revision: 02038fd76fcdcfa1a7161714eb5d2a02828950f5 https://github.com/groonga/groonga/commit/02038fd76fcdcfa1a7161714eb5d2a02828950f5 Merged 714af80: Merge pull request #239 from naoa/support-token_filters-offline-index Message: ii: support token filters for offline index construction Added files: test/command/suite/token_filters/stem/offline_index.expected test/command/suite/token_filters/stem/offline_index.test Modified files: lib/ii.c Modified: lib/ii.c (+4 -1) =================================================================== --- lib/ii.c 2014-11-04 23:35:29 +0900 (411ed65) +++ lib/ii.c 2014-11-05 12:35:44 +0900 (2107969) @@ -6719,9 +6719,10 @@ get_tmp_lexicon(grn_ctx *ctx, grn_ii_buffer *ii_buffer) grn_obj *range = grn_ctx_at(ctx, DB_OBJ(ii_buffer->lexicon)->range); grn_obj *tokenizer; grn_obj *normalizer; + grn_obj *token_filters; grn_obj_flags flags; grn_table_get_info(ctx, ii_buffer->lexicon, &flags, NULL, - &tokenizer, &normalizer, NULL); + &tokenizer, &normalizer, &token_filters); flags &= ~GRN_OBJ_PERSISTENT; tmp_lexicon = grn_table_create(ctx, NULL, 0, NULL, flags, domain, range); if (tmp_lexicon) { @@ -6730,6 +6731,8 @@ get_tmp_lexicon(grn_ctx *ctx, grn_ii_buffer *ii_buffer) GRN_INFO_DEFAULT_TOKENIZER, tokenizer); grn_obj_set_info(ctx, tmp_lexicon, GRN_INFO_NORMALIZER, normalizer); + grn_obj_set_info(ctx, tmp_lexicon, + GRN_INFO_TOKEN_FILTERS, token_filters); if ((flags & GRN_OBJ_TABLE_TYPE_MASK) == GRN_OBJ_TABLE_PAT_KEY) { grn_pat_cache_enable(ctx, (grn_pat *)tmp_lexicon, PAT_CACHE_SIZE); } Added: test/command/suite/token_filters/stem/offline_index.expected (+54 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/token_filters/stem/offline_index.expected 2014-11-05 12:35:44 +0900 (0178cda) @@ -0,0 +1,54 @@ +register token_filters/stem +[[0,0.0,0.0],true] +table_create Memos TABLE_NO_KEY +[[0,0.0,0.0],true] +column_create Memos content COLUMN_SCALAR ShortText +[[0,0.0,0.0],true] +table_create Terms TABLE_PAT_KEY ShortText --default_tokenizer TokenBigram --normalizer NormalizerAuto --token_filters TokenFilterStem +[[0,0.0,0.0],true] +load --table Memos +[ +{"content": "I develop Groonga"}, +{"content": "I'm developing Groonga"}, +{"content": "I developed Groonga"} +] +[[0,0.0,0.0],3] +column_create Terms memos_content COLUMN_INDEX|WITH_POSITION Memos content +[[0,0.0,0.0],true] +select Memos --match_columns content --query "develops" +[ + [ + 0, + 0.0, + 0.0 + ], + [ + [ + [ + 3 + ], + [ + [ + "_id", + "UInt32" + ], + [ + "content", + "ShortText" + ] + ], + [ + 1, + "I develop Groonga" + ], + [ + 2, + "I'm developing Groonga" + ], + [ + 3, + "I developed Groonga" + ] + ] + ] +] Added: test/command/suite/token_filters/stem/offline_index.test (+22 -0) 100644 =================================================================== --- /dev/null +++ test/command/suite/token_filters/stem/offline_index.test 2014-11-05 12:35:44 +0900 (f0788a1) @@ -0,0 +1,22 @@ +#@on-error omit +register token_filters/stem +#@on-error default + +table_create Memos TABLE_NO_KEY +column_create Memos content COLUMN_SCALAR ShortText + +table_create Terms TABLE_PAT_KEY ShortText \ + --default_tokenizer TokenBigram \ + --normalizer NormalizerAuto \ + --token_filters TokenFilterStem + +load --table Memos +[ +{"content": "I develop Groonga"}, +{"content": "I'm developing Groonga"}, +{"content": "I developed Groonga"} +] + +column_create Terms memos_content COLUMN_INDEX|WITH_POSITION Memos content + +select Memos --match_columns content --query "develops" -------------- next part -------------- HTML����������������������������...Download
