make python training and java inference configurable

Author: thygesen

tf-ner-poc/src/main/java/org/apache/opennlp/namefinder/PredictionConfiguration.java

@@ -20,6 +20,7 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.regex.Pattern;
 
 public class PredictionConfiguration {
 
@@ -28,6 +29,11 @@ public class PredictionConfiguration {
   private String vocabTags;
   private String savedModel;
 
+  private boolean useLowerCaseEmbeddings;
+  private boolean allowUNK;
+  private boolean allowNUM;
+  private Pattern digitPattern = Pattern.compile("\\d+(,\\d+)*(\\.\\d+)?");
+
   public PredictionConfiguration(String vocabWords, String vocabChars, String vocabTags, String savedModel) {
     this.vocabWords = vocabWords;
     this.vocabChars = vocabChars;
@@ -51,6 +57,38 @@ public String getSavedModel() {
     return savedModel;
   }
 
+  public boolean isUseLowerCaseEmbeddings() {
+    return useLowerCaseEmbeddings;
+  }
+
+  public void setUseLowerCaseEmbeddings(boolean useLowerCaseEmbeddings) {
+    this.useLowerCaseEmbeddings = useLowerCaseEmbeddings;
+  }
+
+  public boolean isAllowUNK() {
+    return allowUNK;
+  }
+
+  public void setAllowUNK(boolean allowUNK) {
+    this.allowUNK = allowUNK;
+  }
+
+  public boolean isAllowNUM() {
+    return allowNUM;
+  }
+
+  public void setAllowNUM(boolean allowNUM) {
+    this.allowNUM = allowNUM;
+  }
+
+  public Pattern getDigitPattern() {
+    return digitPattern;
+  }
+
+  public void setDigitPattern(Pattern digitPattern) {
+    this.digitPattern = digitPattern;
+  }
+
   public InputStream getVocabWordsInputStream() throws IOException{
     return new FileInputStream(getVocabWords());
   }

tf-ner-poc/src/main/java/org/apache/opennlp/namefinder/SequenceTagging.java

@@ -44,19 +44,23 @@ public SequenceTagging(PredictionConfiguration config) throws IOException {
     model = SavedModelBundle.load(config.getSavedModel(), "serve");
     session = model.session();
 
-    this.wordIndexer = new WordIndexer(new FileInputStream(config.getVocabWords()),
+    this.wordIndexer = new WordIndexer(config.isUseLowerCaseEmbeddings(), config.isAllowNUM(), config.isAllowNUM(),
+            new FileInputStream(config.getVocabWords()),
             new FileInputStream(config.getVocabChars()));
 
+    this.wordIndexer.setDigitPattern(config.getDigitPattern());
+
     this.indexTagger = new IndexTagger((new FileInputStream(config.getVocabTags())));
   }
 
   public SequenceTagging(InputStream modelZipPackage) throws IOException {
 
     Path tmpDir = ModelUtil.writeModelToTmpDir(modelZipPackage);
 
-    try (InputStream wordsIn = Files.newInputStream(tmpDir.resolve("word_dict.txt"));
+    try (InputStream configIn = Files.newInputStream(tmpDir.resolve("config.properties"));
+         InputStream wordsIn = Files.newInputStream(tmpDir.resolve("word_dict.txt"));
          InputStream charsIn = Files.newInputStream(tmpDir.resolve("char_dict.txt"))) {
-      wordIndexer = new WordIndexer(wordsIn, charsIn);
+      wordIndexer = new WordIndexer(configIn, wordsIn, charsIn);
     }
 
     try (InputStream in = Files.newInputStream(tmpDir.resolve("label_dict.txt"))) {
@@ -122,6 +126,10 @@ private String[][] predict(TokenIds tokenIds) {
     }
   }
 
+  public WordIndexer getWordIndexer() {
+    return wordIndexer;
+  }
+
   @Override
   public void clearAdaptiveData() {
   }

tf-ner-poc/src/main/java/org/apache/opennlp/namefinder/WordIndexer.java

@@ -17,30 +17,83 @@
 
 package org.apache.opennlp.namefinder;
 
+import opennlp.tools.util.StringUtil;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Properties;
 import java.util.regex.Pattern;
 
-import opennlp.tools.util.StringUtil;
-
 public class WordIndexer {
 
   private final Map<Character, Integer> char2idx;
   private final Map<String, Integer> word2idx;
 
-  public static String UNK = "$UNK$";
-  public static String NUM = "$NUM$";
+  public static String UNK = "__UNK__";
+  public static String NUM = "__NUM__";
 
   private boolean lowerCase = false;
-  private boolean allowUnk = false;
+  private boolean allowUnk = true;
+  private boolean allowNum = false;
 
   private Pattern digitPattern = Pattern.compile("\\d+(,\\d+)*(\\.\\d+)?");
 
+  public boolean isLowerCase() {
+    return lowerCase;
+  }
+
+  public void setLowerCase(boolean lowerCase) {
+    this.lowerCase = lowerCase;
+  }
+
+  public boolean isAllowUnk() {
+    return allowUnk;
+  }
+
+  public void setAllowUnk(boolean allowUnk) {
+    this.allowUnk = allowUnk;
+  }
+
+  public boolean isAllowNum() {
+    return allowNum;
+  }
+
+  public void setAllowNum(boolean allowNum) {
+    this.allowNum = allowNum;
+  }
+
+  public Pattern getDigitPattern() {
+    return digitPattern;
+  }
+
+  public void setDigitPattern(Pattern digitPattern) {
+    this.digitPattern = digitPattern;
+  }
+
+  public WordIndexer(InputStream config, InputStream vocabWords, InputStream vocabChars) throws IOException {
+    this(vocabWords, vocabChars);
+    Properties props = new Properties();
+    if (config != null) {
+      props.load(new InputStreamReader(config, "UTF8"));
+      this.setLowerCase(Boolean.valueOf(props.getProperty("lower_case_embeddings")));
+      this.setAllowUnk(Boolean.valueOf(props.getProperty("allow_unk")));
+      this.setAllowNum(Boolean.valueOf(props.getProperty("allow_num")));
+      this.setDigitPattern(Pattern.compile(props.getProperty("digit_pattern")));
+    }
+  }
+
+  public WordIndexer(boolean lowerCaseTokens, boolean allowUnk, boolean allowNum, InputStream vocabWords, InputStream vocabChars) throws IOException {
+    this(vocabWords, vocabChars);
+    this.allowUnk = allowUnk;
+    this.allowNum = allowNum;
+    this.lowerCase = lowerCaseTokens;
+  }
+
   public WordIndexer(InputStream vocabWords, InputStream vocabChars) throws IOException {
     this.word2idx = new HashMap<>();
     try(BufferedReader in = new BufferedReader(new InputStreamReader(vocabWords, "UTF8"))) {
@@ -113,8 +166,8 @@ private Ids apply(String word) {
       word = StringUtil.toLowerCase(word);
     }
 
-    // if (digitPattern.matcher(word).find())
-    //  word = NUM;
+    if (allowNum && digitPattern.matcher(word).find())
+      word = NUM;
 
     // 2. get id of word
     Integer wordId;

tf-ner-poc/src/main/python/namefinder/namefinder.py

@@ -20,14 +20,14 @@
 # This poc is based on source code taken from:
 # https://github.com/guillaumegenthial/sequence_tagging
 
-import sys
 from math import floor
 import tensorflow as tf
 import re
 import numpy as np
 import zipfile
 import os
 from tempfile import TemporaryDirectory
+import argparse
 
 # global variables for unknown word and numbers
 __UNK__ = '__UNK__'
@@ -68,12 +68,16 @@ def __str__(self):
 class NameFinder:
     label_dict = {}
 
-    def __init__(self, use_lower_case_embeddings=False, vector_size=100):
+    def __init__(self, use_lower_case_embeddings, allow_unk, allow_num, digit_pattern, encoding, vector_size=100):
         self.__vector_size = vector_size
         self.__use_lower_case_embeddings = use_lower_case_embeddings
+        self.__allow_unk = allow_unk
+        self.__allow_num = allow_num
+        self.__digit_pattern = re.compile(digit_pattern)
+        self.__encoding = encoding
 
     def load_data(self, word_dict, file):
-        with open(file) as f:
+        with open(file, encoding=self.__encoding) as f:
             raw_data = f.readlines()
 
         sentences = []
@@ -96,7 +100,8 @@ def load_data(self, word_dict, file):
                 if self.__use_lower_case_embeddings:
                     token = token.lower()
 
-                # TODO: implement NUM encoding
+                if self.__allow_num and self.__digit_pattern.match(token):
+                    token = __NUM__
 
                 if word_dict.get(token) is not None:
                     vector = word_dict[token]
@@ -340,8 +345,8 @@ def write_mapping(tags, output_filename):
             f.write('{}\n'.format(tag))
 
 
-def load_glove(glove_file):
-    with open(glove_file) as f:
+def load_glove(glove_file, encoding='utf-8'):
+    with open(glove_file, encoding=encoding) as f:
 
         word_dict = {}
         embeddings = []
@@ -381,16 +386,28 @@ def load_glove(glove_file):
 
 
 def main():
-    if len(sys.argv) != 5:
-        print("Usage namefinder.py embedding_file train_file dev_file test_file")
-        return
-
-    word_dict, rev_word_dict, embeddings, vector_size = load_glove(sys.argv[1])
-
-    name_finder = NameFinder(vector_size)
-
-    sentences, labels, char_set = name_finder.load_data(word_dict, sys.argv[2])
-    sentences_dev, labels_dev, char_set_dev = name_finder.load_data(word_dict, sys.argv[3])
+    parser = argparse.ArgumentParser()
+    parser.add_argument("embedding_file", help="path to the embeddings file.")
+    parser.add_argument("train_file", help="path to the training file.")
+    parser.add_argument("dev_file", help="path to the dev file.")
+    parser.add_argument("--allow_unk", help="use general UNK vector for unknown tokens.", default=True)
+    parser.add_argument("--allow_num", help="use general NUM vector for all numeric tokens.", default=False)
+    parser.add_argument("--lower_case_embeddings", help="convert tokens to lowercase for embeddings lookup.",
+                        default=False)
+    parser.add_argument("--digit_pattern", help="regex to use for identifying numeric tokens.",
+                        default='^\\d+(,\\d+)*(\\.\\d+)?$')
+    parser.add_argument("--data_encoding", help="set encoding of train and dev data.", default='utf-8')
+    parser.add_argument("--embeddings_encoding", help="set encoding of the embeddings.", default='utf-8')
+    args = parser.parse_args()
+
+    word_dict, rev_word_dict, embeddings, vector_size = load_glove(args.embedding_file, args.embeddings_encoding)
+
+    name_finder = NameFinder(use_lower_case_embeddings=args.lower_case_embeddings, allow_unk=args.allow_unk,
+                             allow_num=args.allow_num, digit_pattern=args.digit_pattern,
+                             encoding=args.data_encoding, vector_size=vector_size)
+
+    sentences, labels, char_set = name_finder.load_data(word_dict, args.train_file)
+    sentences_dev, labels_dev, char_set_dev = name_finder.load_data(word_dict, args.dev_file)
 
     char_dict = {k: v for v, k in enumerate(char_set | char_set_dev)}
 
@@ -472,6 +489,14 @@ def main():
                     write_mapping(name_finder.label_dict, temp_model_dir + "/label_dict.txt")
                     write_mapping(char_dict, temp_model_dir + "/char_dict.txt")
 
+                    write_mapping({'lower_case_embeddings=' + str(args.lower_case_embeddings).lower(): 0,
+                                   'allow_unk=' + str(args.allow_unk).lower(): 1,
+                                   'allow_num=' + str(args.allow_num).lower(): 2,
+                                   'digit_pattern=' + re.escape(args.digit_pattern): 3,
+                                   'data_encoding=' + args.data_encoding: 4,
+                                   'embeddings_encoding=' + args.embeddings_encoding: 5},
+                                  temp_model_dir + "/config.properties")
+
                     zipf = zipfile.ZipFile("namefinder-" + str(epoch) + ".zip", 'w', zipfile.ZIP_DEFLATED)
 
                     for root, dirs, files in os.walk(temp_model_dir):