feat: add cache for tokenize

azu · azu · commit dc17f9a99fae · 2020-12-21T14:14:35.000+09:00
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,22 @@
+name: test
+on: [push, pull_request]
+env:
+  CI: true
+jobs:
+  test:
+    name: "Test on Node.js ${{ matrix.node-version }}"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [10, 12, 14]
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: setup Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Install
+        run: yarn install
+      - name: Test
+        run: yarn test
diff --git a/.mocharc.json b/.mocharc.json
@@ -0,0 +1,5 @@
+{
+  "require": [
+    "ts-node-test-register"
+  ]
+}
diff --git a/.travis.yml b/.travis.yml
diff --git a/README.md b/README.md
@@ -1,11 +1,13 @@
-# kuromojin [![Build Status](https://travis-ci.org/azu/kuromojin.svg?branch=master)](https://travis-ci.org/azu/kuromojin)
+# kuromojin [![Actions Status: test](https://github.com/azu/kuromojin/workflows/test/badge.svg)](https://github.com/azu/kuromojin/actions?query=workflow%3A"test")
 
 Provide a high level wrapper for [kuromoji.js](https://github.com/takuyaa/kuromoji.js "kuromoji.js").
 
 ## Features
 
 - Promise based API
 - Cache Layer
+    - Fetch the dictionary at once
+    - Return same tokens for same text
 
 ## Installation
 
diff --git a/package.json b/package.json
@@ -34,19 +34,19 @@
     "netlify": "mkdir -p public && cp -r node_modules/kuromoji/dict public/"
   },
   "dependencies": {
-    "kuromoji": "0.1.1"
+    "kuromoji": "^0.1.1"
   },
   "devDependencies": {
-    "@types/mocha": "^5.2.7",
-    "@types/node": "^12.0.12",
-    "cross-env": "^5.2.0",
-    "husky": "^3.0.0",
-    "lint-staged": "^9.1.0",
-    "mocha": "^6.1.4",
-    "prettier": "^1.18.2",
-    "ts-node": "^8.3.0",
-    "ts-node-test-register": "^8.0.1",
-    "typescript": "^3.5.2"
+    "@types/mocha": "^8.2.0",
+    "@types/node": "^14.14.14",
+    "cross-env": "^7.0.3",
+    "husky": "^4.3.6",
+    "lint-staged": "^10.5.3",
+    "mocha": "^8.2.1",
+    "prettier": "^2.2.1",
+    "ts-node": "^9.1.1",
+    "ts-node-test-register": "^9.0.1",
+    "typescript": "^4.1.3"
   },
   "email": "azuciao@gmail.com",
   "prettier": {
diff --git a/src/Deferred.ts b/src/Deferred.ts
@@ -2,10 +2,11 @@
 "use strict";
 export default class Deferred<T> {
     promise: Promise<T>;
-    resolve!: (value?: T) => void;
-    reject!: (reason?: any) => void;
+    resolve!: (value: T) => void;
+    reject!: (reason: any) => void;
+
     constructor() {
-        this.promise = new Promise((resolve, reject) => {
+        this.promise = new Promise<T>((resolve, reject) => {
             this.resolve = resolve;
             this.reject = reject;
         });
diff --git a/src/kuromojin.ts b/src/kuromojin.ts
@@ -66,14 +66,17 @@ const getNodeModuleDirPath = () => {
 
 // cache for tokenizer
 let _tokenizer: null | Tokenizer = null;
+let tokenizeCacheMap = new Map<string, KuromojiToken[]>()
 // lock boolean
 let isLoading = false;
 
 export type getTokenizerOption = {
     dicPath: string;
+    // Default: false
+    noCache?: boolean
 };
 
-export function getTokenizer(options: getTokenizerOption = {dicPath: getNodeModuleDirPath()}): Promise<Tokenizer> {
+export function getTokenizer(options: getTokenizerOption = { dicPath: getNodeModuleDirPath() }): Promise<Tokenizer> {
     if (_tokenizer) {
         return Promise.resolve(_tokenizer);
     }
@@ -92,8 +95,18 @@ export function getTokenizer(options: getTokenizerOption = {dicPath: getNodeModu
     return deferred.promise;
 }
 
-export function tokenize(text: string, options?: getTokenizerOption) {
+export function tokenize(text: string, options?: getTokenizerOption): Promise<KuromojiToken[]> {
     return getTokenizer(options).then(tokenizer => {
-        return tokenizer.tokenizeForSentence(text);
+        if (options?.noCache) {
+            return tokenizer.tokenizeForSentence(text);
+        } else {
+            const cache = tokenizeCacheMap.get(text);
+            if (cache) {
+                return cache;
+            }
+            const tokens = tokenizer.tokenizeForSentence(text);
+            tokenizeCacheMap.set(text, tokens);
+            return tokens;
+        }
     });
 }
diff --git a/test/mocha.opts b/test/mocha.opts
diff --git a/yarn.lock b/yarn.lock

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +{
 +  "require": [
 +    "ts-node-test-register"
 +  ]
 +}