Kouhei Sutou
null+****@clear*****
Wed May 9 17:53:04 JST 2018
Kouhei Sutou 2018-05-09 17:53:04 +0900 (Wed, 09 May 2018) New Revision: e045de2b5598c4e2947eae1a9f6910f2896da455 https://github.com/groonga/groonga/commit/e045de2b5598c4e2947eae1a9f6910f2896da455 Message: token: support overlap information Modified files: include/groonga/token.h lib/grn_token.h lib/token.c lib/tokenizers.c Modified: include/groonga/token.h (+7 -0) =================================================================== --- include/groonga/token.h 2018-05-09 15:07:13 +0900 (8783a4607) +++ include/groonga/token.h 2018-05-09 17:53:04 +0900 (5dc45a4f2) @@ -146,6 +146,13 @@ GRN_API grn_rc grn_token_set_source_length(grn_ctx *ctx, grn_token *token, uint32_t length); +GRN_API grn_bool +grn_token_is_overlap(grn_ctx *ctx, + grn_token *token); +GRN_API grn_rc +grn_token_set_overlap(grn_ctx *ctx, + grn_token *token, + grn_bool is_overlap); #ifdef __cplusplus } /* extern "C" */ Modified: lib/grn_token.h (+1 -0) =================================================================== --- lib/grn_token.h 2018-05-09 15:07:13 +0900 (7ba2a3afb) +++ lib/grn_token.h 2018-05-09 17:53:04 +0900 (5d256a396) @@ -29,6 +29,7 @@ struct _grn_token { grn_token_status status; uint64_t source_offset; uint32_t source_length; + grn_bool is_overlap; }; grn_rc grn_token_init(grn_ctx *ctx, grn_token *token); Modified: lib/token.c (+31 -0) =================================================================== --- lib/token.c 2018-05-09 15:07:13 +0900 (b1599f85c) +++ lib/token.c 2018-05-09 17:53:04 +0900 (abbac818d) @@ -26,6 +26,7 @@ grn_token_init(grn_ctx *ctx, grn_token *token) token->status = GRN_TOKEN_CONTINUE; token->source_offset = 0; token->source_length = 0; + token->is_overlap = GRN_FALSE; GRN_API_RETURN(ctx->rc); } @@ -171,6 +172,34 @@ exit: GRN_API_RETURN(ctx->rc); } +grn_bool +grn_token_is_overlap(grn_ctx *ctx, grn_token *token) +{ + GRN_API_ENTER; + if (!token) { + ERR(GRN_INVALID_ARGUMENT, + "[token][overlap][get] token must not be NULL"); + GRN_API_RETURN(0); + } + GRN_API_RETURN(token->is_overlap); +} + +grn_rc +grn_token_set_overlap(grn_ctx *ctx, + grn_token *token, + grn_bool is_overlap) +{ + GRN_API_ENTER; + if (!token) { + ERR(GRN_INVALID_ARGUMENT, + "[token][overlap][set] token must not be NULL"); + goto exit; + } + token->is_overlap = is_overlap; +exit: + GRN_API_RETURN(ctx->rc); +} + grn_rc grn_token_reset(grn_ctx *ctx, grn_token *token) { @@ -183,6 +212,7 @@ grn_token_reset(grn_ctx *ctx, grn_token *token) token->status = GRN_TOKEN_CONTINUE; token->source_offset = 0; token->source_length = 0; + token->is_overlap = GRN_FALSE; exit: GRN_API_RETURN(ctx->rc); } @@ -204,6 +234,7 @@ grn_token_copy(grn_ctx *ctx, token->status = source->status; token->source_offset = source->source_offset; token->source_length = source->source_length; + token->is_overlap = source->is_overlap; exit: GRN_API_RETURN(ctx->rc); } Modified: lib/tokenizers.c (+1 -0) =================================================================== --- lib/tokenizers.c 2018-05-09 15:07:13 +0900 (b9c578a8c) +++ lib/tokenizers.c 2018-05-09 17:53:04 +0900 (be7d9bca5) @@ -839,6 +839,7 @@ ngram_next(grn_ctx *ctx, } grn_token_set_data(ctx, token, p, data_size); grn_token_set_status(ctx, token, status); + grn_token_set_overlap(ctx, token, tokenizer->overlap); if (checks) { size_t i; uint32_t uncount_offset = 0; -------------- next part -------------- HTML����������������������������... URL: https://lists.osdn.me/mailman/archives/groonga-commit/attachments/20180509/e6b57d94/attachment-0001.htm
