Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+en.all.raw filter=lfs diff=lfs merge=lfs -text
+en.train.raw filter=lfs diff=lfs merge=lfs -text
+tools/mosesdecoder/contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1 filter=lfs diff=lfs merge=lfs -text
+tools/mosesdecoder/contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1.binphr.tgtdata.wa filter=lfs diff=lfs merge=lfs -text
+tools/mosesdecoder/moses/TranslationModel/UG/util/ibm1-align filter=lfs diff=lfs merge=lfs -text

en.all.raw

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abe998aaaae6f542f7856cdeeda0fc02da818c50387a42a3888c2464d44ebbd6
+size 17279365577

en.train.raw

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcb606760557d54ad120c38d8c170ce7ad52b2e63a1e77f9f81439123ff3ee8b
+size 17276590615

tools/fastBPE/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2019 Guillaume Lample,Timothée Lacroix
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

tools/fastBPE/MANIFEST.in

@@ -0,0 +1,3 @@
+include fastBPE/*.cc
+include fastBPE/*.hpp
+include fastBPE/*.pyx

tools/fastBPE/README.md

@@ -0,0 +1,83 @@
+
+# fastBPE
+
+C++ implementation of [Neural Machine Translation of Rare Words with Subword Units](https://arxiv.org/abs/1508.07909), with Python API.
+
+## Installation
+
+Compile with:
+```
+g++ -std=c++11 -pthread -O3 fastBPE/main.cc -IfastBPE -o fast
+```
+
+## Usage:
+
+### List commands
+```
+./fast
+usage: fastbpe <command> <args>
+
+The commands supported by fastBPE are:
+
+getvocab input1 [input2]             extract the vocabulary from one or two text files
+learnbpe nCodes input1 [input2]      learn BPE codes from one or two text files
+applybpe output input codes [vocab]  apply BPE codes to a text file
+applybpe_stream codes [vocab]        apply BPE codes to stdin and outputs to stdout
+```
+
+fastBPE also supports stdin inputs. For instance, these two commands are equivalent:
+```
+./fast getvocab text > vocab
+cat text | ./fast getvocab - > vocab
+```
+But the first one will memory map the input file to read it efficiently, which can be more than twice faster than stdin on very large files. Similarly, these two commands are equivalent:
+```
+./fast applybpe output input codes vocab
+cat input | ./fast applybpe_stream codes vocab > output
+```
+Although the first one will be significantly faster on large datasets, as it uses multi-threading to pre-compute the BPE splits of all words in the input file.
+
+### Learn codes
+```
+./fast learnbpe 40000 train.de train.en > codes
+```
+
+### Apply codes to train
+```
+./fast applybpe train.de.40000 train.de codes
+./fast applybpe train.en.40000 train.en codes
+```
+
+### Get train vocabulary
+```
+./fast getvocab train.de.40000 > vocab.de.40000
+./fast getvocab train.en.40000 > vocab.en.40000
+```
+
+### Apply codes to valid and test
+```
+./fast applybpe valid.de.40000 valid.de codes vocab.de.40000
+./fast applybpe valid.en.40000 valid.en codes vocab.en.40000
+./fast applybpe test.de.40000  test.de  codes vocab.de.40000
+./fast applybpe test.en.40000  test.en  codes vocab.en.40000
+```
+
+## Python API
+
+To install the Python API, simply run:
+```bash
+python setup.py install
+```
+
+**Note:** For Mac OSX Users, add `export MACOSX_DEPLOYMENT_TARGET=10.x` (x=9 or 10, depending on your version) or `-stdlib=libc++` to the `extra_compile_args` of `setup.py` before/during the above install command, as appropriate.
+
+Call the API using:
+
+```python
+import fastBPE
+
+bpe = fastBPE.fastBPE(codes_path, vocab_path)
+bpe.apply(["Roasted barramundi fish", "Centrally managed over a client-server architecture"])
+
+>> ['Ro@@ asted barr@@ am@@ un@@ di fish', 'Centr@@ ally managed over a cli@@ ent-@@ server architecture']
+```

tools/fastBPE/fastBPE/fastBPE.hpp

@@ -0,0 +1,692 @@
+#pragma once
+
+#include <algorithm>
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <list>
+#include <set>
+#include <stdio.h>
+#include <string>
+#include <cstring>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <thread>
+#include <unistd.h> // ftruncate
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+
+namespace fastBPE {
+
+using namespace std;
+
+const size_t kMaxPairs = 1000 * 1000 * 1000;
+const size_t kThreads = max(1, min(10, int(thread::hardware_concurrency())));
+const char *kEndWord = "</w>";
+const size_t kEndWordLength = 4;
+const char *kTokenDelim = "@@";
+const size_t kTokenDelimLength = 2;
+
+int safeOpen(const char *file_path, int flags, mode_t mode = 0) {
+  int fd = open(file_path, flags, mode);
+  if (fd < 0) {
+    fprintf(stderr, "Cannot open text file %s\n", file_path);
+    exit(EXIT_FAILURE);
+  }
+  return fd;
+}
+
+void readText(const char *fp, unordered_map<string, uint32_t> &word_count) {
+  string cur_word;
+  uint64_t total = 0;
+  auto deal_with_char = [&](char cur_char){
+    if (cur_char == ' ' || cur_char == '\n') {
+      if (cur_word.size() == 0)
+        return;
+      // end of word
+      auto it = word_count.find(cur_word);
+      int count = it != word_count.end() ? it->second : 0;
+      word_count[cur_word] = count + 1;
+      total++;
+      cur_word.clear();
+    } else {
+      cur_word.push_back(cur_char);
+    }
+  };
+
+  if (string(fp).compare("-") == 0) {
+    for (std::string line; std::getline(std::cin, line);) {
+      for(char c: line){
+        deal_with_char(c);
+      }
+      deal_with_char('\n');
+    }
+  }
+  else {
+    int fd = safeOpen(fp, O_RDONLY);
+
+    struct stat s;
+    fstat(fd, &s);
+    fprintf(stderr, "Loading vocabulary from %s ...\n", fp);
+
+    size_t size = s.st_size;
+    char *f = (char *)mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
+
+    for (size_t i = 0; i < size; i++) {
+      deal_with_char(f[i]);
+    }
+  }
+  fprintf(stderr, "Read %lu words (%lu unique) from text file.\n", total,
+          word_count.size());
+}
+
+std::pair<size_t, uint64_t> output_or_count(
+  unordered_map<string, string> &bpe, size_t size, char *f, char *fo
+) {
+  string cur_word;
+  size_t charOut = 0;
+  uint64_t total = 0;
+  for (size_t i = 0; i < size; i++) {
+    auto &cur_char = f[i];
+    if (cur_char == ' ' || cur_char == '\n') {
+      if (cur_word.size() == 0) {
+        if (fo != nullptr) fo[charOut] = cur_char;
+        charOut++;
+        continue;
+      }
+      // end of word : write bpe to output
+      auto it = bpe.find(cur_word);
+      assert(it != bpe.end());
+      for (auto x : it->second) {
+        if (fo != nullptr) fo[charOut] = x;
+        charOut++;
+      }
+      if (fo != nullptr) fo[charOut] = cur_char;
+      charOut++;
+
+      total++;
+      cur_word.clear();
+    } else {
+      cur_word.push_back(cur_char);
+    }
+  }
+  return std::make_pair(charOut, total);
+}
+
+void outputText(const char *fpo, const char *fp,
+                unordered_map<string, string> &bpe) {
+
+  int fd = safeOpen(fp, O_RDONLY);
+  auto fdOut = safeOpen(fpo, O_RDWR | O_CREAT | O_TRUNC, 0666);
+
+  struct stat s;
+  fstat(fd, &s);
+
+  fprintf(stderr, "Applying BPE to %s ...\n", fp);
+  auto size = s.st_size;
+  char *f = (char *)mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
+
+  auto p = output_or_count(bpe, size, f, nullptr);
+  size_t out_size = p.first;
+
+  if (ftruncate(fdOut, out_size) < 0) {
+    fprintf(stderr, "Couldn't truncate output file %s to size %lu\n", fpo,
+            out_size);
+    exit(EXIT_FAILURE);
+  }
+
+
+  char *fo = (char *)mmap(NULL, out_size, PROT_WRITE, MAP_SHARED, fdOut, 0);
+  if (fo == MAP_FAILED) {
+    fprintf(stderr, "Output memory map failed : %d.\n", errno);
+    exit(EXIT_FAILURE);
+  }
+  p = output_or_count(bpe, size, f, fo);
+  fprintf(stderr, "Modified %lu words from text file.\n", p.second);
+  munmap(fo, out_size);
+  munmap(f, size);
+  close(fdOut);
+  close(fd);
+}
+
+struct pair_hash {
+  template <class T1, class T2> size_t operator()(const pair<T1, T2> &p) const {
+    auto h1 = hash<T1>{}(p.first);
+    auto h2 = hash<T2>{}(p.second);
+    size_t seed = h1;
+    // boost::hash_combine
+    return h2 + 0x9e3779b9 + (seed << 6) + (seed >> 2);
+  }
+};
+
+void tokenize(const unordered_map<string, uint32_t> &word_count,
+              unordered_map<string, uint32_t> &token_to_int,
+              vector<string> &int_to_token, vector<list<uint32_t>> &words,
+              vector<int32_t> &counts) {
+
+  for (auto &x : word_count) {
+    auto &word = x.first;
+
+    words.push_back(list<uint32_t>());
+    auto &current_word = words.back();
+    counts.push_back(x.second);
+
+    int pos = 0, realLength = 0;
+    int lastStart = 0;
+    while (word[pos]) {
+      bool newChar = (word[pos] & 0xc0) != 0x80; // not a continuation byte
+      realLength += newChar;
+      // new token
+      if (newChar && pos > 0) {
+        auto new_token = word.substr(lastStart, pos - lastStart);
+        if (token_to_int.count(new_token) == 0) {
+          int_to_token.push_back(new_token);
+          token_to_int[new_token] = int_to_token.size() - 1;
+        }
+        current_word.push_back(token_to_int[new_token]);
+        lastStart = pos;
+      }
+      pos++;
+    }
+    auto new_token = word.substr(lastStart, string::npos) + kEndWord;
+    if (token_to_int.count(new_token) == 0) {
+      int_to_token.push_back(new_token);
+      token_to_int[new_token] = int_to_token.size() - 1;
+    }
+    current_word.push_back(token_to_int[new_token]);
+  }
+}
+
+void tokenize_str(const unordered_map<string, uint32_t> &word_count,
+                  unordered_map<string, vector<string>> &words) {
+
+  for (auto &x : word_count) {
+    auto &word = x.first;
+    words[word] = vector<string>();
+
+    int pos = 0, realLength = 0;
+    int lastStart = 0;
+    while (word[pos]) {
+      bool newChar = (word[pos] & 0xc0) != 0x80; // not a continuation byte
+      realLength += newChar;
+      // new token
+      if (newChar && pos > 0) {
+        auto new_token = word.substr(lastStart, pos - lastStart);
+        words[word].push_back(new_token);
+        lastStart = pos;
+      }
+      pos++;
+    }
+    auto new_token = word.substr(lastStart, string::npos) + kEndWord;
+    words[word].push_back(new_token);
+  }
+}
+
+using tp = pair<uint32_t, uint32_t>;
+using tps = pair<string, string>;
+using pc = unordered_map<tp, pair<int32_t, tp> *, pair_hash>;
+
+void count_in_word(
+    list<uint32_t> &word, uint32_t wi, uint32_t count, pc &pair_counts,
+    vector<pair<int32_t, tp>> &contiguous_counts,
+    unordered_map<tp, unordered_set<uint32_t>, pair_hash> &where) {
+  bool second = false;
+  tp cur_pair;
+  for (uint32_t token : word) {
+    if (second) {
+      cur_pair.first = cur_pair.second;
+    }
+    cur_pair.second = token;
+    if (second) {
+      auto it = pair_counts.find(cur_pair);
+      if (it == pair_counts.end()) {
+        contiguous_counts.emplace_back(0, cur_pair);
+        auto *added = &contiguous_counts.back();
+        pair_counts.emplace(piecewise_construct, forward_as_tuple(cur_pair),
+                            forward_as_tuple(added));
+        where[cur_pair].emplace();
+      }
+      if (count > 0) {
+        where[cur_pair].insert(wi);
+      } else {
+        where[cur_pair].erase(wi);
+      }
+      pair_counts[cur_pair]->first += count;
+    } else {
+      second = true;
+    }
+  }
+}
+
+void find_maxp(vector<pair<int32_t, tp>> &contiguous_counts, tp &maxp,
+               int32_t &max_c) {
+  max_c = 0;
+  for (auto &x : contiguous_counts) {
+    if (x.first > max_c) {
+      max_c = x.first;
+      maxp = x.second;
+    } else if (x.first == max_c and x.second < maxp) {
+      maxp = x.second;
+    }
+  }
+}
+
+void getvocab(const char *inputFile1, const char *inputFile2) {
+  // get vocab
+  unordered_map<string, uint32_t> word_count;
+  readText(inputFile1, word_count);
+  if (strcmp(inputFile2, "") != 0) {
+    readText(inputFile2, word_count);
+  }
+
+  // sort vocab
+  auto compFunctor = [](pair<string, int> elem1, pair<string, int> elem2) {
+    return elem1.second > elem2.second ||
+           (elem1.second == elem2.second && elem1.first < elem2.first);
+  };
+  set<pair<string, int>, decltype(compFunctor)> sorted_vocab(
+      word_count.begin(), word_count.end(), compFunctor);
+  assert(word_count.size() == sorted_vocab.size());
+
+  // print sorted vocab
+  for (auto element : sorted_vocab)
+    cout << element.first << " " << element.second << endl;
+}
+
+void learnbpe(const uint32_t kNPairs, const char *inputFile1,
+              const char *inputFile2) {
+  // get vocab
+  unordered_map<string, uint32_t> word_count;
+  readText(inputFile1, word_count);
+  if (strcmp(inputFile2, "") != 0) {
+    readText(inputFile2, word_count);
+  }
+
+  // a token is an int, it represents a string
+  unordered_map<string, uint32_t> token_to_int;
+  vector<string> int_to_token;
+
+  vector<list<uint32_t>> words;
+  vector<int32_t> counts;
+
+  tokenize(word_count, token_to_int, int_to_token, words, counts);
+
+  vector<pair<int32_t, tp>> contiguous_counts;
+  contiguous_counts.reserve(kMaxPairs);
+
+  pc pair_counts;
+  unordered_map<tp, unordered_set<uint32_t>, pair_hash> where_to_update;
+
+  tp cur_pair;
+  int32_t max_c = 0;
+  tp max_p;
+  for (uint32_t wi = 0; wi < words.size(); wi++) {
+    count_in_word(words[wi], wi, counts[wi], pair_counts, contiguous_counts,
+                  where_to_update);
+  }
+  find_maxp(contiguous_counts, max_p, max_c);
+  for (size_t i = 0; i < kNPairs; i++) {
+    // create new token for pair. replace
+    auto new_token = int_to_token[max_p.first] + int_to_token[max_p.second];
+    cout << int_to_token[max_p.first] << " " << int_to_token[max_p.second]
+         << " " << max_c << endl;
+
+    uint32_t new_token_id = int_to_token.size();
+    int_to_token.push_back(new_token);
+    token_to_int[new_token] = new_token_id;
+    max_c = 0;
+    auto change_count = [&](tp pair, int32_t v, uint32_t wi) {
+      auto it = pair_counts.find(pair);
+      if (it != pair_counts.end()) {
+        // assert(it->second + v >= 0);
+        it->second->first += v;
+      } else {
+        if (v > 0) {
+          contiguous_counts.emplace_back(v, pair);
+          pair_counts.emplace(piecewise_construct, forward_as_tuple(pair),
+                              forward_as_tuple(&(contiguous_counts.back())));
+          where_to_update[pair] = unordered_set<uint32_t>();
+        }
+      }
+      if (v > 0)
+        where_to_update[pair].insert(wi);
+    };
+
+    for (auto wi : where_to_update[max_p]) {
+      auto &cur_word = words[wi];
+      auto it = cur_word.begin();
+      bool second = false;
+      while (it != cur_word.end()) {
+        if (second) {
+          cur_pair.first = cur_pair.second;
+        }
+        cur_pair.second = *it;
+
+        if (second) {
+          // found the pair
+          if (cur_pair == max_p) {
+            it--; // points to first element of pair
+            // if there is a token before us
+            if (it != cur_word.begin()) {
+              it--;
+              change_count(make_pair(*it, cur_pair.first), -counts[wi], wi);
+              change_count(make_pair(*it, new_token_id), counts[wi], wi);
+              it++;
+            }
+
+            it = cur_word.insert(it, new_token_id); // it points to new token
+            it++;                    // it points to first element of pair
+            it = cur_word.erase(it); // it points to second element of pair
+            it = cur_word.erase(it); // it points to next value
+
+            // if there is a token after the one we inserted
+            if (it != cur_word.end()) {
+              change_count(make_pair(cur_pair.second, *it), -counts[wi], wi);
+              change_count(make_pair(new_token_id, *it), counts[wi], wi);
+            }
+            cur_pair.second = new_token_id;
+          } else {
+            it++;
+          }
+        } else {
+          second = true;
+          it++;
+        }
+      }
+    }
+
+    if (pair_counts.find(max_p) != pair_counts.end()){
+      pair_counts[max_p]->first = 0;
+    }
+    find_maxp(contiguous_counts, max_p, max_c);
+  }
+}
+
+void split(vector<string> &splits, const string &text, char sep) {
+  size_t start = 0, end = 0;
+  while ((end = text.find(sep, start)) != string::npos) {
+    if (end != start)
+      splits.push_back(text.substr(start, end - start));
+    start = end + 1;
+  }
+  if (end != start && start < text.size())
+    splits.push_back(text.substr(start));
+}
+
+void readVocab(const char *fp, unordered_map<string, uint32_t> &vocab) {
+  ifstream file(fp);
+  if (!file) {
+    fprintf(stderr, "Cannot open vocabulary file %s\n", fp);
+    exit(EXIT_FAILURE);
+  }
+  fprintf(stderr, "Loading vocabulary from %s ...\n", fp);
+  string line;
+  uint64_t total = 0;
+  while (getline(file, line)) {
+    vector<string> splits;
+    split(splits, line, ' ');
+    assert(splits.size() == 2);
+    assert(vocab.find(splits[0]) == vocab.end());
+    int count = stoi(splits[1]);
+    vocab[splits[0]] = count;
+    total += count;
+  }
+  fprintf(stderr, "Read %lu words (%lu unique) from vocabulary file.\n", total,
+          vocab.size());
+}
+
+void readCodes(const char *fp, unordered_map<tps, uint32_t, pair_hash> &codes,
+               unordered_map<string, tps> &reversed_codes) {
+  ifstream file(fp);
+  if (!file) {
+    fprintf(stderr, "Cannot open codes file %s\n", fp);
+    exit(EXIT_FAILURE);
+  }
+  fprintf(stderr, "Loading codes from %s ...\n", fp);
+  string line;
+  while (getline(file, line)) {
+    vector<string> splits;
+    split(splits, line, ' ');
+    assert(splits.size() == 3);
+    auto pair = make_pair(splits[0], splits[1]);
+    string concat = splits[0] + splits[1];
+    assert(codes.find(pair) == codes.end());
+    assert(reversed_codes.find(concat) == reversed_codes.end());
+    codes[pair] = codes.size();
+    reversed_codes[concat] = pair;
+  }
+  fprintf(stderr, "Read %lu codes from the codes file.\n", codes.size());
+}
+
+void decompose(const string s, vector<string> &newSubwords,
+               const unordered_map<string, tps> &reversed_codes,
+               const unordered_map<string, uint32_t> &vocab, bool isFinal) {
+  auto it = reversed_codes.find(s);
+  if (it == reversed_codes.end()) {
+    // TODO this whole block below is just some sanity check
+    // if we cannot un-merge a subword, it has to be a char
+    string s2 = isFinal ? s.substr(0, s.size() - kEndWordLength) : s;
+    int count = 0;
+    for (size_t j = 0; j < s2.size(); j++) {
+      if ((s2[j] & 0xc0) != 0x80) {
+        count++;
+      }
+    }
+    assert(count == 1);
+    newSubwords.push_back(s);
+    return;
+  }
+  assert(it != reversed_codes.end());
+  string token1 = it->second.first;
+  if (vocab.find(token1 + kTokenDelim) == vocab.end()) {
+    decompose(token1, newSubwords, reversed_codes, vocab, false);
+  } else {
+    newSubwords.push_back(token1);
+  }
+  string token2 = it->second.second;
+  auto query = token2 + kTokenDelim;
+  if (isFinal) {
+    query = token2.substr(0, token2.size() - kEndWordLength);
+  }
+  if (vocab.find(query) == vocab.end()) {
+    decompose(token2, newSubwords, reversed_codes, vocab, isFinal);
+  } else {
+    newSubwords.push_back(token2);
+  }
+}
+
+void limitVocab(const vector<string> &subwords, vector<string> &newSubwords,
+                const unordered_map<string, tps> &reversed_codes,
+                const unordered_map<string, uint32_t> &vocab) {
+  string query;
+  for (size_t i = 0; i < subwords.size(); i++) {
+    bool isFinal = i == subwords.size() - 1;
+    auto &subword = subwords[i];
+    if (isFinal) {
+      query = subword.substr(0, subword.size() - kEndWordLength);
+    } else {
+      query = subword + kTokenDelim;
+    }
+    if (vocab.find(query) == vocab.end()) {
+      decompose(subword, newSubwords, reversed_codes, vocab, isFinal);
+    } else {
+      newSubwords.push_back(subword);
+    }
+  }
+}
+
+string process_bpe(vector<string> &subwords,
+                   unordered_map<tps, uint32_t, pair_hash> &codes,
+                   unordered_map<string, tps> &reversed_codes,
+                   unordered_map<string, uint32_t> &vocab) {
+  // merge subWords as much as possible
+  vector<string> newSubwords;
+  while (subwords.size() > 1) {
+    // find the best pair
+    int bestPairId = -1;
+    auto bestPair = codes.end(); // TODO ugly hack that works
+    for (size_t i = 0; i < subwords.size() - 1; i++) {
+      auto pair = make_pair(subwords[i], subwords[i + 1]);
+      auto it = codes.find(pair);
+      int pairRank = it == codes.end() ? -1 : it->second;
+      if (pairRank >= 0 && (bestPairId == -1 || int(bestPair->second) > pairRank)) {
+        bestPair = it;
+        bestPairId = i;
+      }
+    }
+    // if we cannot merge anything, stop
+    if (bestPairId == -1) {
+      break;
+    }
+    // otherwise, merge subWords
+    bool justMerged = false;
+    newSubwords = vector<string>();
+    for (size_t i = 0; i < subwords.size(); i++) {
+      if ((i + 1 < subwords.size()) && (not justMerged) &&
+          subwords[i] == bestPair->first.first &&
+          subwords[i + 1] == bestPair->first.second) {
+        newSubwords.push_back(subwords[i] + subwords[i + 1]);
+        justMerged = true;
+      } else {
+        if (not justMerged) {
+          newSubwords.push_back(subwords[i]);
+        }
+        justMerged = false;
+      }
+    }
+    subwords = newSubwords;
+  }
+  // check that we are only using words in the dictionary
+  if (vocab.size() > 0) {
+    vector<string> newSubwords;
+    limitVocab(subwords, newSubwords, reversed_codes, vocab);
+    subwords = newSubwords;
+  }
+  // concat subWords
+  string result;
+  for (auto x : subwords) {
+    result = result + x + kTokenDelim + " ";
+  }
+  return result.substr(
+    0,
+    result.size() - kEndWordLength - kTokenDelimLength - 1 // "</w>@@ "
+  );
+}
+
+void applybpe(const char *outputFile, const char *inputFile,
+              const char *codesPath, const char *vocabPath) {
+  // read vocabulary (to which we want to limit the output file)
+  unordered_map<string, uint32_t> vocab;
+  if (strcmp(vocabPath, "") != 0) {
+    readVocab(vocabPath, vocab);
+  }
+
+  // read codes
+  unordered_map<tps, uint32_t, pair_hash> codes;
+  unordered_map<string, tps> reversed_codes;
+  readCodes(codesPath, codes, reversed_codes);
+
+  // read input file words
+  unordered_map<string, uint32_t> word_count;
+  readText(inputFile, word_count);
+
+  // tokenize
+  unordered_map<string, vector<string>> bpeTok;
+  tokenize_str(word_count, bpeTok);
+
+  vector<pair<string, vector<string>>> bpeTokVec;
+  for (auto x : bpeTok) {
+    bpeTokVec.push_back(x);
+  }
+
+  // apply BPE codes to each word
+  unordered_map<string, string> bpe[kThreads];
+  vector<thread> threads;
+  for (size_t i = 0; i < kThreads; i++) {
+    threads.emplace_back(
+      [&](size_t this_thread) {
+        for (size_t w = this_thread; w < bpeTokVec.size(); w += kThreads) {
+          auto &x = bpeTokVec[w];
+          bpe[this_thread][x.first] = process_bpe(x.second, codes, reversed_codes, vocab);
+        }
+      },
+      i
+    );
+  }
+
+  unordered_map<string, string> final_bpe;
+  for (size_t i = 0; i < kThreads; i++) {
+    threads[i].join();
+    for (auto x : bpe[i]) {
+      final_bpe[x.first] = x.second;
+    }
+  }
+  // output
+  outputText(outputFile, inputFile, final_bpe);
+}
+
+
+class BPEApplyer {
+private:
+  unordered_map<string, uint32_t> vocab;
+  unordered_map<tps, uint32_t, pair_hash> codes;
+  unordered_map<string, tps> reversed_codes;
+
+public:
+  BPEApplyer(const string& codesPath, const string& vocabPath) {
+    if (vocabPath.size() > 0) readVocab(vocabPath.c_str(), vocab);
+    readCodes(codesPath.c_str(), codes, reversed_codes);
+  }
+
+  vector<string> apply(vector<string>& sentences) {
+    vector<string> res;
+    for(auto &s: sentences) {
+      res.emplace_back("");
+      string& cur = res.back();
+      vector<string> words;
+      split(words, s, ' ');
+      for (size_t i = 0; i < words.size(); i++) {
+        auto word = words[i];
+        vector<string> word_bpes;
+        int pos = 0, realLength = 0;
+        int lastStart = 0;
+        while (word[pos]) {
+          bool newChar = (word[pos] & 0xc0) != 0x80; // not a continuation byte
+          realLength += newChar;
+          if (newChar && pos > 0) {
+            auto new_token = word.substr(lastStart, pos - lastStart);
+            word_bpes.push_back(new_token);
+            lastStart = pos;
+          }
+          pos++;
+        }
+        auto bpe = word.substr(lastStart, string::npos) + kEndWord;
+        word_bpes.push_back(bpe);
+        cur += process_bpe(word_bpes, codes, reversed_codes, vocab);
+        if (i < words.size() - 1) cur += " ";
+      }
+    }
+    return res;
+  }
+
+};
+
+
+void applybpe_stream(const char *codesPath, const char *vocabPath) {
+  BPEApplyer applyer(codesPath, vocabPath);
+  std::string line;
+  while(std::getline(std::cin, line)) {
+    vector<string> tmp;
+    tmp.push_back(line);
+    for(auto& l : applyer.apply(tmp)){
+      std::cout << l << std::endl;
+    }
+  }
+}
+
+};

tools/fastBPE/fastBPE/fastBPE.pyx

@@ -0,0 +1,24 @@
+# cython: language_level=3
+# distutils: language = c++
+
+from libcpp.vector cimport vector
+from libcpp.string cimport string
+
+cdef extern from "fastBPE.hpp" namespace "fastBPE":
+    cdef cppclass BPEApplyer:
+        BPEApplyer(const string& codes_path, const string& vocab_path)
+        vector[string] apply(vector[string]& sentences)
+
+cdef class fastBPE:
+    cdef BPEApplyer* c_obj
+
+    def __dealloc__(self):
+        del self.c_obj
+
+    def __init__(self, codes_path, vocab_path=""):
+        self.c_obj = new BPEApplyer(codes_path.encode(), vocab_path.encode())
+
+    def apply(self, sentences):
+        cdef vector[string] s = [x.encode() for x in sentences]
+        cdef vector[string] res = self.c_obj.apply(s)
+        return [x.decode() for x in res]

tools/fastBPE/fastBPE/main.cc

@@ -0,0 +1,43 @@
+#include "fastBPE.hpp"
+
+using namespace std;
+using namespace fastBPE;
+
+void printUsage() {
+  cerr
+      << "usage: fastbpe <command> <args>\n\n"
+      << "The commands supported by fastBPE are:\n\n"
+      << "getvocab input1 [input2]             extract the vocabulary from one "
+         "or two text files\n"
+      << "learnbpe nCodes input1 [input2]      learn BPE codes from one or two "
+         "text files\n"
+      << "applybpe output input codes [vocab]  apply BPE codes to a text file\n"
+      << "applybpe_stream codes [vocab]        apply BPE codes to stdin and output to stdout\n"
+      << endl;
+}
+
+
+int main(int argc, char **argv) {
+  if (argc < 2) {
+    printUsage();
+    exit(EXIT_FAILURE);
+  }
+  string command = argv[1];
+  if (command == "getvocab") {
+    assert(argc == 3 || argc == 4);
+    getvocab(argv[2], argc == 4 ? argv[3] : "");
+  } else if (command == "learnbpe") {
+    assert(argc == 4 || argc == 5);
+    learnbpe(stoi(argv[2]), argv[3], argc == 5 ? argv[4] : "");
+  } else if (command == "applybpe") {
+    assert(argc == 5 || argc == 6);
+    applybpe(argv[2], argv[3], argv[4], argc == 6 ? argv[5] : "");
+  } else if (command == "applybpe_stream") {
+    assert(argc == 3 || argc == 4);
+    applybpe_stream(argv[2], argc == 4 ? argv[3] : "");
+  } else {
+    printUsage();
+    exit(EXIT_FAILURE);
+  }
+  return 0;
+}

tools/fastBPE/setup.py

@@ -0,0 +1,49 @@
+from setuptools import setup, find_packages, Extension
+from distutils.command.sdist import sdist as _sdist
+
+
+try:
+    from Cython.Build import cythonize
+except ImportError:
+    use_cython = False
+else:
+    use_cython = True
+
+
+if use_cython:
+    extension = 'pyx'
+else:
+    extension = 'cpp'
+
+
+extensions = [
+    Extension(
+        'fastBPE',
+        [ "fastBPE/fastBPE." + extension ],
+        language='c++',
+        extra_compile_args=[
+            "-std=c++11", "-Ofast", "-pthread"
+        ],
+    ),
+]
+if use_cython:
+    extensions = cythonize(extensions)
+
+
+with open('README.md') as f:
+    readme = f.read()
+
+
+setup(
+    name = 'fastBPE',
+    version = '0.1.1',
+    description = 'C++ implementation of Neural Machine Translation of Rare Words with Subword Units, with Python API.',
+    url = 'https://github.com/glample/fastBPE',
+    long_description = readme,
+    long_description_content_type = 'text/markdown',
+    ext_package = '',
+    ext_modules = extensions,
+    packages=[
+        'fastBPE',
+    ],
+)

tools/mosesdecoder/.beautify-ignore

@@ -0,0 +1,38 @@
+# Files and directories that beautify.py should not clean up.
+#
+# This file is not as advanced as, say, .gitignore.  It only supports files
+# and directory paths relative to the project root, one per line, no globs,
+# no quotes.
+#
+# Leading and trailing whitespace is stripped from filenames, but internal
+# whitespace is preserved.
+#
+# Lines starting with a hash mark, such as this one, are comments.  The hash
+# mark must be the first character on the line.  Blank lines are ignored.
+#
+# The .beautify-ignore file must be encoded in UTF-8.
+
+boost
+contrib
+irstlm
+jam-files
+lm
+mingw/MosesGUI/icons_rc.py
+mingw/MosesGUI/Ui_credits.py
+mingw/MosesGUI/Ui_mainWindow.py
+moses/TranslationModel/UG
+moses/server
+moses/parameters
+moses/thread_safe_container.h
+phrase-extract/pcfg-common
+phrase-extract/syntax-common
+randlm
+# Filename suffixes in here are language codes, so e.g. ".pl" means
+# Polish, not Perl.
+scripts/share/nonbreaking_prefixes
+search
+srilm
+util
+xmlrpc-c
+.git
+util/ug_cache_with_timeout.h

tools/mosesdecoder/.gitignore

@@ -0,0 +1,90 @@
+tools
+*.d
+*.pyc
+*.lo
+*.o
+*.so
+*.lo
+*.o
+*.la
+*.a
+*.swp
+*.save
+*.cmd
+*~
+*.gch
+dist*
+jam-files/bjam
+jam-files/engine/bootstrap
+jam-files/engine/bin.*
+lm/build_binary
+lm/query
+mert/evaluator
+mert/extractor
+mert/hgdecode
+mert/mert
+mert/megam_i686.opt
+mert/pro
+mert/kbmira
+misc/processLexicalTable
+misc/processPhraseTable
+misc/queryLexicalTable
+mira/mira
+mira/Makefile
+mira/Makefile.in
+misc/queryPhraseTable
+moses-chart-cmd/src/moses_chart
+moses-cmd/src/checkplf
+moses-cmd/src/lmbrgrid
+moses-cmd/src/moses
+regression-testing/moses-reg-test-data-*
+regression-testing/tests/mert.extractor-bin/FEATSTAT*
+regression-testing/tests/mert.extractor-bin/SCORESTAT*
+scripts/ems/biconcor/biconcor
+scripts/release-exclude
+scripts/training/cmert-0.5/mert
+scripts/training/compact-rule-table/tools/compactify
+scripts/training/eppex/counter
+scripts/training/eppex/eppex
+scripts/training/lexical-reordering/score
+scripts/training/memscore/memscore
+scripts/training/mbr/mbr
+scripts/training/phrase-extract/consolidate
+scripts/training/phrase-extract/consolidate-direct
+scripts/training/phrase-extract/consolidate-reverse
+scripts/training/phrase-extract/extract
+scripts/training/phrase-extract/extract-ghkm/tools/extract-ghkm
+scripts/training/phrase-extract/extract-lex
+scripts/training/phrase-extract/extract-rules
+scripts/training/phrase-extract/relax-parse
+scripts/training/phrase-extract/score
+scripts/training/phrase-extract/statistics
+scripts/training/symal/symal
+dist
+bin
+previous.sh
+contrib/other-builds/*.xcodeproj/project.xcworkspace/
+contrib/other-builds/*.xcodeproj/xcuserdata/
+*/*.xcodeproj/project.xcworkspace
+*/*.xcodeproj/xcuserdata
+
+mert/sentence-bleu
+mert/sentence-bleu-nbest
+._*
+.DS_Store
+*.pbxuser
+*.mode1v3
+
+*.exe
+build/
+nbproject/
+
+mingw/MosesGUI/MosesGUI.e4p
+mingw/MosesGUI/_eric4project/
+
+contrib/m4m/merge-sorted
+mert/hgdecode
+.bash_history*
+doxygen.conf
+doxy
+opt

tools/mosesdecoder/.gitmodules

@@ -0,0 +1,9 @@
+[submodule "contrib/arrow-pipelines/python/pcl"]
+	path = contrib/arrow-pipelines/python/pcl
+	url = https://github.com/ianj-als/pcl.git
+[submodule "contrib/omtc/omtc"]
+	path = contrib/omtc/omtc
+	url = https://github.com/ianj-als/omtc.git
+[submodule "regtest"]
+	path = regtest
+	url = https://github.com/moses-smt/moses-regression-tests

tools/mosesdecoder/.travis.yml

@@ -0,0 +1,24 @@
+sudo: false
+dist: trusty
+language: c
+compiler: gcc
+env:
+  matrix: 
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - subversion
+      - automake
+      - libtool
+      - zlib1g-dev
+      - libbz2-dev
+      - liblzma-dev
+      - libboost-all-dev
+      - libgoogle-perftools-dev
+      - libxmlrpc-c++.*-dev
+      - cmake
+      - csh
+script:
+- ./bjam -j4

tools/mosesdecoder/COPYING

@@ -0,0 +1,460 @@
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+	51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations
+below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it
+becomes a de-facto standard.  To achieve this, non-free programs must
+be allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control
+compilation and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at least
+    three years, to give the same user the materials specified in
+    Subsection 6a, above, for a charge no more than the cost of
+    performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply, and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License
+may add an explicit geographical distribution limitation excluding those
+countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+

tools/mosesdecoder/Jamroot

@@ -0,0 +1,345 @@
+#BUILDING MOSES
+
+#PACKAGES
+#Language models (optional):
+#--with-irstlm=/path/to/irstlm 
+#--with-srilm=/path/to/srilm See moses/LM/Jamfile for more options.
+#--with-maxent-srilm=true (requires a maxent-enabled version of SRILM to be specified via --with-srilm)
+#--with-nplm=/path/to/nplm
+#--with-randlm=/path/to/randlm
+#KenLM is always compiled.  
+#
+#--with-boost=/path/to/boost
+#If Boost is in a non-standard location, specify it here.  This directory is
+#expected to contain include and lib or lib64.  
+#
+#--with-xmlrpc-c=/path/to/xmlrpc-c for libxmlrpc-c (used by server)
+#Note that, like language models, this is the --prefix where the library was
+#installed, not some executable within the library.  
+#
+#--no-xmlrpc-c
+# Don't use xmlrpc-c library, even if it exists. Don't build moses server
+#
+#Compact phrase table and compact lexical reordering table
+#--with-cmph=/path/to/cmph
+#
+#Thread-caching malloc (if present, used for multi-threaded builds by default)
+#--without-tcmalloc does not compile with tcmalloc even if present
+#--full-tcmalloc links against the full version (useful for memory profiling)
+#
+#REGRESSION TESTING
+#--with-regtest=/path/to/moses-reg-test-data
+#
+#INSTALLATION
+#--prefix=/path/to/prefix sets the install prefix [default is source root].
+#--bindir=/path/to/prefix/bin sets the bin directory [PREFIX/bin]
+#--libdir=/path/to/prefix/lib sets the lib directory [PREFIX/lib]
+#--includedir=/path/to/prefix/include installs headers.  
+#  Does not install if missing.  No argument defaults to PREFIX/include .
+#--install-scripts=/path/to/scripts copies scripts into a directory.
+#  Does not install if missing.  No argument defaults to PREFIX/scripts .
+#--git appends the git revision to the prefix directory.
+#
+#
+#BUILD OPTIONS
+# By default, the build is multi-threaded, optimized, and statically linked.  
+# Pass these to change the build:
+#
+# threading=single|multi         controls threading (default multi)
+#
+# variant=release|debug|profile  builds optimized (default), for debug, or for
+#                                profiling
+#
+# link=static|shared             controls preferred linking (default static)
+# --static                       forces static linking (the default will fall
+#                                back to shared)
+#
+# debug-symbols=on|off           include or exclude (default) debugging 
+#                                information also known as -g
+# --notrace                      compiles without TRACE macros
+#
+# --enable-boost-pool            uses Boost pools for the memory SCFG tabgle
+#
+# --enable-mpi                   switch on mpi
+# --without-libsegfault          does not link with libSegFault
+#
+# --max-kenlm-order              maximum ngram order that kenlm can process (default 6)
+#
+# --max-factors                  maximum number of factors (default 4)
+#
+# --unlabelled-source            ignore source labels (redundant in hiero or string-to-tree system)
+#                                for better performance
+#CONTROLLING THE BUILD
+#-a to build from scratch
+#-j$NCPUS to compile in parallel
+#--clean to clean
+#--debug-build to build with Og. Only available with gcc 4.8+
+
+import os ;
+import option ;
+import modules ;
+import path ;
+path-constant TOP : . ;
+
+include $(TOP)/jam-files/sanity.jam ;
+
+home = [ os.environ "HOME" ] ;
+if [ path.exists $(home)/moses-environment.jam ] 
+{ 
+  # for those of use who don't like typing in command line bjam options all day long
+  include $(home)/moses-environment.jam ; 
+}
+include $(TOP)/jam-files/check-environment.jam ; # get resource locations 
+                                                 # from environment variables
+include $(TOP)/jam-files/xmlrpc-c.jam ; # xmlrpc-c stuff for the server
+# include $(TOP)/jam-files/curlpp.jam ;   # curlpp stuff for bias lookup (MMT only)
+
+# exit "done" : 0 ;
+
+max-order = [ option.get "max-kenlm-order" : 6 : 6 ] ;
+if ! [ option.get "max-kenlm-order" ] 
+{
+  # some classes in Moses pull in header files from KenLM, so this needs to be 
+  # defined here, not in moses/lm/Jamfile 
+  option.set "max-kenlm-order" : 6 ;
+  requirements += <define>KENLM_MAX_ORDER=$(max-order) ;
+}
+# exit "all done" : 0 ; 
+
+boost 104400 ;
+external-lib z ;
+
+#lib dl : : <runtime-link>static:<link>static <runtime-link>shared:<link>shared ;
+#requirements += <library>dl ;
+requirements += <cxxflags>-std=c++0x ;
+
+# Allow moses to report the git commit hash of the version used for compilation
+moses_githash = [ _shell "git describe --dirty" ] ;
+requirements += <define>MOSES_VERSION_ID=\\\"$(moses_githash)\\\" ;
+
+if ! [ option.get "without-tcmalloc" : : "yes" ] && [ test_library "tcmalloc_minimal" ] {
+  if [ option.get "full-tcmalloc" : : "yes" ] {
+    external-lib unwind ;
+    external-lib tcmalloc_and_profiler : : unwind ;
+    requirements += <library>tcmalloc_and_profiler <library>unwind <cflags>-fno-omit-frame-pointer <cxxflags>-fno-omit-frame-pointer ;
+  } else {
+    external-lib tcmalloc_minimal ;
+    requirements += <threading>multi:<library>tcmalloc_minimal ;
+  }
+} else {
+  echo "Tip: install tcmalloc for faster threading.  See BUILD-INSTRUCTIONS.txt for more information." ;
+}
+
+if [ option.get "filter-warnings" : : "yes" ] {
+  # given the low coding standards in Moses, we may want to filter out
+  # warnings about poor coding practice that no-one is ever going to fix 
+  # anyway ...
+  requirements += <cxxflags>-Wno-deprecated ;
+  requirements += <cxxflags>-Wno-reorder ;
+  requirements += <cxxflags>-Wno-sign-compare ;
+  requirements += <cxxflags>-Wno-unused-but-set-variable ;
+  requirements += <cxxflags>-Wno-unused-result ;
+  requirements += <cxxflags>-Wno-unused-variable ;
+  requirements += <cxxflags>-Wno-comment ;
+  requirements += <cxxflags>-Wno-strict-aliasing ;
+  requirements += <cxxflags>-Wno-overloaded-virtual ;
+}
+
+if [ option.get "debug-build" : : "yes" ] {
+   requirements += <cxxflags>-Og ;
+   echo "Building with -Og to enable easier profiling and debugging. Only available on gcc 4.8+." ;
+}
+
+if [ option.get "with-address-sanitizer" : : "yes" ] {
+  requirements += <cxxflags>-fsanitize=address ;
+  requirements += <cxxflags>-fno-omit-frame-pointer ;
+  requirements += <linkflags>-fsanitize=address ;
+  echo "Building with AddressSanitizer to enable debugging of memory errors. Only available on gcc 4.8+." ;
+}
+
+if [ option.get "enable-mpi" : : "yes" ] {
+  import mpi ;
+  using mpi ;
+  external-lib boost_mpi ;
+  external-lib boost_serialization ;
+  requirements += <define>MPI_ENABLE ;
+  requirements += <library>mpi ;
+  requirements += <library>boost_mpi ;
+  requirements += <library>boost_serialization ;
+}
+
+mmt = [ option.get "mmt" ] ; 
+if $(mmt) {
+  requirements += <define>MMT ;
+  requirements += <include>$(mmt) ; 
+  mmt_githash = [ _shell "cd $(mmt) && git describe --dirty" ] ;
+  requirements += <define>MMT_VERSION_ID=\\\"$(mmt_githash)\\\" ;
+}
+
+requirements += [ option.get "notrace" : <define>TRACE_ENABLE=1 ] ;
+requirements += [ option.get "enable-boost-pool" : : <define>USE_BOOST_POOL ] ;
+requirements += [ option.get "with-mm" : : <define>PT_UG ] ;
+requirements += [ option.get "with-mm" : : <define>MAX_NUM_FACTORS=4 ] ;
+requirements += [ option.get "unlabelled-source" : : <define>UNLABELLED_SOURCE ] ;
+
+if [ option.get "with-oxlm" ] {
+  external-lib gomp ;
+  requirements += <library>boost_serialization ;
+  requirements += <library>gomp ;
+}
+
+if [ option.get "with-cmph" : : "yes" ] {
+  requirements += <define>HAVE_CMPH ;
+}
+
+if [ option.get "with-icu" : : "yes" ]
+{
+  external-lib icuuc ;
+  external-lib icuio ;
+  external-lib icui18n ;
+  requirements += <library>icuuc/<link>shared ;
+  requirements += <library>icuio/<link>shared ;
+  requirements += <library>icui18n/<link>shared ;
+  requirements += <cxxflags>-fPIC ;
+  requirements += <address-model>64 ;
+#  requirements += <runtime-link>shared ;
+}
+
+# for probing pt
+external-lib boost_serialization ;
+requirements += <library>boost_serialization/<runtime-link>static ;
+
+if [ option.get "with-vw" ] {
+  requirements += <define>HAVE_VW ;
+}
+
+project : default-build
+  <threading>multi
+  <warnings>on
+  <debug-symbols>off
+  <variant>release
+  <link>static
+  ;
+
+#Apparently OS X likes to link against iconv for fgetsUTF8.
+lib iconv ;
+requirements += <os>MACOSX:<library>iconv ;
+
+project : requirements 
+  <threading>multi:<define>WITH_THREADS
+  <threading>multi:<library>boost_thread
+  <library>boost_system
+  <library>boost_program_options
+  <define>_FILE_OFFSET_BITS=64 <define>_LARGE_FILES
+  $(requirements)
+  <include>.
+  ;
+
+
+#Add directories here if you want their incidental targets too (i.e. tests).
+build-projects lm util phrase-extract phrase-extract/syntax-common search moses moses/LM mert moses-cmd scripts regression-testing ;
+# contrib/mira
+
+if [ option.get "with-mm-extras" : : "yes" ]
+{
+  alias mm-extras :
+  moses/TranslationModel/UG//bitext-find 
+  moses/TranslationModel/UG//ptable-describe-features 
+  moses/TranslationModel/UG//count-ptable-features 
+  moses/TranslationModel/UG//ptable-sigtest-filter 
+  moses/TranslationModel/UG//ptable-lookup 
+  moses/TranslationModel/UG//ptable-lookup-corpus 
+  moses/TranslationModel/UG//check-coverage 
+  moses/TranslationModel/UG/mm//mtt-demo1 
+  moses/TranslationModel/UG/mm//mtt-dump 
+  moses/TranslationModel/UG/mm//mam2symal 
+  moses/TranslationModel/UG/mm//mam_verify 
+  moses/TranslationModel/UG/mm//mmlex-lookup 
+  moses/TranslationModel/UG/mm//mtt-count-words 
+  moses/TranslationModel/UG/mm//calc-coverage 
+  moses/TranslationModel/UG//try-align 
+  ;
+} 
+else
+{
+  alias mm-extras ;
+}
+
+if [ option.get "with-mm" : : "yes" ]
+{
+ alias mm :  
+  moses/TranslationModel/UG/mm//mtt-build 
+  moses/TranslationModel/UG/mm//symal2mam 
+  moses/TranslationModel/UG/mm//mmlex-build 
+  ;
+}
+else
+{
+ alias mm ; 
+}
+
+if [ option.get "with-rephraser" : : "yes" ]
+{
+  alias rephraser :
+  contrib/rephraser//paraphrase
+  ;
+}
+else
+{
+  alias rephraser ;
+}
+
+alias programs : 
+lm//programs 
+moses-cmd//programs 
+OnDiskPt//CreateOnDiskPt 
+OnDiskPt//queryOnDiskPt 
+mert//programs 
+misc//programs 
+symal 
+phrase-extract 
+phrase-extract//lexical-reordering 
+phrase-extract//extract-ghkm 
+phrase-extract//pcfg-extract 
+phrase-extract//pcfg-score 
+phrase-extract//extract-mixed-syntax 
+phrase-extract//score-stsg
+phrase-extract//filter-rule-table
+phrase-extract//postprocess-egret-forests
+biconcor 
+# contrib/mira//mira 
+contrib/server//mosesserver 
+mm
+mm-extras
+rephraser
+contrib/c++tokenizer//tokenizer
+contrib/expected-bleu-training//train-expected-bleu
+contrib/expected-bleu-training//prepare-expected-bleu-training
+
+probingpt//programs
+moses2//programs
+;
+
+
+install-bin-libs programs ;
+install-headers headers-base : [ path.glob-tree biconcor contrib lm mert misc moses-cmd OnDiskPt phrase-extract symal util : *.hh *.h ] : . ;
+install-headers headers-moses : moses//headers-to-install : moses ;
+
+alias install : prefix-bin prefix-lib headers-base headers-moses ;
+
+if ! [ option.get "includedir" : : $(prefix)/include ] {
+  explicit install headers-base headers-moses ;
+}
+
+if [ path.exists $(TOP)/dist ] && $(prefix) != dist {
+  echo "You have a $(TOP)/dist directory, but the build system now places files directly in the root i.e. $(TOP)/bin ." ;
+  echo "To disable this message, delete $(TOP)/dist ." ;
+  echo ;
+}
+
+#local temp = [ _shell "bash source ./s.sh" ] ;
+local temp = [ _shell "mkdir -p $(PREFIX)/bin" ] ;
+local temp = [ _shell "rm -f $(PREFIX)/bin/moses_chart" ] ;
+local temp = [ _shell "cd $(PREFIX)/bin && ln -sf moses moses_chart" ] ;
+local temp = [ _shell "cd $(PREFIX)/bin && ln -sf CreateProbingPT CreateProbingPT2" ] ;
+

tools/mosesdecoder/OnDiskPt/Jamfile

@@ -0,0 +1,5 @@
+fakelib OnDiskPt : OnDiskWrapper.cpp SourcePhrase.cpp TargetPhrase.cpp Word.cpp Phrase.cpp PhraseNode.cpp TargetPhraseCollection.cpp Vocab.cpp OnDiskQuery.cpp ../moses//headers ;
+
+exe CreateOnDiskPt : Main.cpp ..//boost_filesystem ../moses//moses OnDiskPt ;
+exe queryOnDiskPt : queryOnDiskPt.cpp ..//boost_filesystem ../moses//moses OnDiskPt ;
+

tools/mosesdecoder/OnDiskPt/Main.cpp

@@ -0,0 +1,273 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <iterator>
+#include <cassert>
+#include "moses/InputFileStream.h"
+#include "moses/Timer.h"
+#include "moses/Util.h"
+#include "OnDiskWrapper.h"
+#include "SourcePhrase.h"
+#include "TargetPhrase.h"
+#include "TargetPhraseCollection.h"
+#include "Word.h"
+#include "Vocab.h"
+#include "Main.h"
+
+using namespace std;
+using namespace OnDiskPt;
+
+int main (int argc, char * const argv[])
+{
+  // insert code here...
+  Moses::ResetUserTime();
+  Moses::PrintUserTime("Starting");
+
+  if (argc != 8) {
+    std::cerr << "Usage: " << argv[0] << " numSourceFactors numTargetFactors numScores tableLimit sortScoreIndex inputPath outputPath" << std::endl;
+    return 1;
+  }
+
+  int numSourceFactors	= Moses::Scan<int>(argv[1])
+                          , numTargetFactors	= Moses::Scan<int>(argv[2])
+                              , numScores				= Moses::Scan<int>(argv[3])
+                                  , tableLimit				= Moses::Scan<int>(argv[4]);
+  TargetPhraseCollection::s_sortScoreInd			= Moses::Scan<int>(argv[5]);
+  assert(TargetPhraseCollection::s_sortScoreInd < numScores);
+
+  const string filePath 	= argv[6]
+                            ,destPath	= argv[7];
+
+  Moses::InputFileStream inStream(filePath);
+
+  OnDiskWrapper onDiskWrapper;
+  onDiskWrapper.BeginSave(destPath, numSourceFactors, numTargetFactors, numScores);
+
+  PhraseNode &rootNode = onDiskWrapper.GetRootSourceNode();
+  size_t lineNum = 0;
+  string line;
+
+  while(getline(inStream, line)) {
+    lineNum++;
+    if (lineNum%1000 == 0) cerr << "." << flush;
+    if (lineNum%10000 == 0) cerr << ":" << flush;
+    if (lineNum%100000 == 0) cerr << lineNum << flush;
+    //cerr << lineNum << " " << line << endl;
+
+    std::vector<float> misc(1);
+    SourcePhrase sourcePhrase;
+    TargetPhrase *targetPhrase = new TargetPhrase(numScores);
+    OnDiskPt::PhrasePtr spShort = Tokenize(sourcePhrase, *targetPhrase, line, onDiskWrapper, numScores, misc);
+    assert(misc.size() == onDiskWrapper.GetNumCounts());
+
+    rootNode.AddTargetPhrase(sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, misc, spShort);
+  }
+
+  rootNode.Save(onDiskWrapper, 0, tableLimit);
+  onDiskWrapper.EndSave();
+
+  Moses::PrintUserTime("Finished");
+
+  //pause();
+  return 0;
+
+} // main()
+
+bool Flush(const OnDiskPt::SourcePhrase *prevSourcePhrase, const OnDiskPt::SourcePhrase *currSourcePhrase)
+{
+  if (prevSourcePhrase == NULL)
+    return false;
+
+  assert(currSourcePhrase);
+  bool ret = (*currSourcePhrase > *prevSourcePhrase);
+  //cerr << *prevSourcePhrase << endl << *currSourcePhrase << " " << ret << endl << endl;
+
+  return ret;
+}
+
+OnDiskPt::PhrasePtr Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, const std::string &lineStr, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc)
+{
+  char line[lineStr.size() + 1];
+  strcpy(line, lineStr.c_str());
+
+  stringstream sparseFeatures, property;
+
+  size_t scoreInd = 0;
+
+  // MAIN LOOP
+  size_t stage = 0;
+  /*	0 = source phrase
+   1 = target phrase
+   2 = scores
+   3 = align
+   4 = count
+   7 = properties
+   */
+  char *tok = strtok (line," ");
+  OnDiskPt::PhrasePtr out(new Phrase());
+  while (tok != NULL) {
+    if (0 == strcmp(tok, "|||")) {
+      ++stage;
+    } else {
+      switch (stage) {
+      case 0: {
+        WordPtr w = Tokenize(sourcePhrase, tok, true, true, onDiskWrapper, 1);
+        if (w != NULL)
+          out->AddWord(w);
+
+        break;
+      }
+      case 1: {
+        Tokenize(targetPhrase, tok, false, true, onDiskWrapper, 0);
+        break;
+      }
+      case 2: {
+        float score = Moses::Scan<float>(tok);
+        targetPhrase.SetScore(score, scoreInd);
+        ++scoreInd;
+        break;
+      }
+      case 3: {
+        //targetPhrase.Create1AlignFromString(tok);
+        targetPhrase.CreateAlignFromString(tok);
+        break;
+      }
+      case 4: {
+        // store only the 3rd one (rule count)
+        float val = Moses::Scan<float>(tok);
+        misc[0] = val;
+        break;
+      }
+      case 5: {
+        // sparse features
+        sparseFeatures << tok << " ";
+        break;
+      }
+      case 6: {
+        property << tok << " ";
+        break;
+      }
+      default:
+        cerr << "ERROR in line " << line << endl;
+        assert(false);
+        break;
+      }
+    }
+
+    tok = strtok (NULL, " ");
+  } // while (tok != NULL)
+
+  assert(scoreInd == numScores);
+  targetPhrase.SetSparseFeatures(Moses::Trim(sparseFeatures.str()));
+  targetPhrase.SetProperty(Moses::Trim(property.str()));
+  targetPhrase.SortAlign();
+  return out;
+} // Tokenize()
+
+OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase
+                           , const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm
+                           , OnDiskPt::OnDiskWrapper &onDiskWrapper, int retSourceTarget)
+{
+  // retSourceTarget: 0 = don't return anything. 1 = source, 2 = target
+
+  bool nonTerm = false;
+  size_t tokSize = token.size();
+  int comStr =token.compare(0, 1, "[");
+
+  if (comStr == 0) {
+    comStr = token.compare(tokSize - 1, 1, "]");
+    nonTerm = comStr == 0;
+  }
+
+  OnDiskPt::WordPtr out;
+  if (nonTerm) {
+    // non-term
+    size_t splitPos		= token.find_first_of("[", 2);
+    string wordStr	= token.substr(0, splitPos);
+
+    if (splitPos == string::npos) {
+      // lhs - only 1 word
+      WordPtr word(new Word());
+      word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
+      phrase.AddWord(word);
+    } else {
+      // source & target non-terms
+      if (addSourceNonTerm) {
+        WordPtr word(new Word());
+        word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
+        phrase.AddWord(word);
+
+        if (retSourceTarget == 1) {
+          out = word;
+        }
+      }
+
+      wordStr = token.substr(splitPos, tokSize - splitPos);
+      if (addTargetNonTerm) {
+        WordPtr word(new Word());
+        word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
+        phrase.AddWord(word);
+
+        if (retSourceTarget == 2) {
+          out = word;
+        }
+      }
+
+    }
+  } else {
+    // term
+    WordPtr word(new Word());
+    word->CreateFromString(token, onDiskWrapper.GetVocab());
+    phrase.AddWord(word);
+    out = word;
+  }
+
+  return out;
+}
+
+void InsertTargetNonTerminals(std::vector<std::string> &sourceToks, const std::vector<std::string> &targetToks, const ::AlignType &alignments)
+{
+  for (int ind = alignments.size() - 1; ind >= 0; --ind) {
+    const ::AlignPair &alignPair = alignments[ind];
+    size_t sourcePos = alignPair.first
+                       ,targetPos = alignPair.second;
+
+    const string &target = targetToks[targetPos];
+    sourceToks.insert(sourceToks.begin() + sourcePos + 1, target);
+
+  }
+}
+
+class AlignOrderer
+{
+public:
+  bool operator()(const ::AlignPair &a, const ::AlignPair &b) const {
+    return a.first < b.first;
+  }
+};
+
+void SortAlign(::AlignType &alignments)
+{
+  std::sort(alignments.begin(), alignments.end(), AlignOrderer());
+}

tools/mosesdecoder/OnDiskPt/Main.h

@@ -0,0 +1,39 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include "SourcePhrase.h"
+#include "TargetPhrase.h"
+
+typedef std::pair<size_t, size_t>  AlignPair;
+typedef std::vector<AlignPair> AlignType;
+
+OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase
+                           , const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm
+                           , OnDiskPt::OnDiskWrapper &onDiskWrapper, int retSourceTarget);
+OnDiskPt::PhrasePtr Tokenize(OnDiskPt::SourcePhrase &sourcePhrase, OnDiskPt::TargetPhrase &targetPhrase
+                             , const std::string &lineStr, OnDiskPt::OnDiskWrapper &onDiskWrapper
+                             , int numScores
+                             , std::vector<float> &misc);
+
+void InsertTargetNonTerminals(std::vector<std::string> &sourceToks, const std::vector<std::string> &targetToks, const AlignType &alignments);
+void SortAlign(AlignType &alignments);
+bool Flush(const OnDiskPt::SourcePhrase *prevSource, const OnDiskPt::SourcePhrase *currSource);
+

tools/mosesdecoder/OnDiskPt/OnDiskQuery.cpp

@@ -0,0 +1,83 @@
+#include "OnDiskQuery.h"
+
+namespace OnDiskPt
+{
+
+void OnDiskQuery::Tokenize(Phrase &phrase,
+                           const std::string &token,
+                           bool addSourceNonTerm,
+                           bool addTargetNonTerm)
+{
+  bool nonTerm = false;
+  size_t tokSize = token.size();
+  int comStr =token.compare(0, 1, "[");
+
+  if (comStr == 0) {
+    comStr = token.compare(tokSize - 1, 1, "]");
+    nonTerm = comStr == 0;
+  }
+
+  if (nonTerm) {
+    // non-term
+    size_t splitPos = token.find_first_of("[", 2);
+    std::string wordStr = token.substr(0, splitPos);
+
+    if (splitPos == std::string::npos) {
+      // lhs - only 1 word
+      WordPtr word (new Word());
+      word->CreateFromString(wordStr, m_wrapper.GetVocab());
+      phrase.AddWord(word);
+    } else {
+      // source & target non-terms
+      if (addSourceNonTerm) {
+        WordPtr word( new Word());
+        word->CreateFromString(wordStr, m_wrapper.GetVocab());
+        phrase.AddWord(word);
+      }
+
+      wordStr = token.substr(splitPos, tokSize - splitPos);
+      if (addTargetNonTerm) {
+        WordPtr word(new Word());
+        word->CreateFromString(wordStr, m_wrapper.GetVocab());
+        phrase.AddWord(word);
+      }
+
+    }
+  } else {
+    // term
+    WordPtr word(new Word());
+    word->CreateFromString(token, m_wrapper.GetVocab());
+    phrase.AddWord(word);
+  }
+}
+
+SourcePhrase OnDiskQuery::Tokenize(const std::vector<std::string>& tokens)
+{
+  SourcePhrase sourcePhrase;
+  if (tokens.size() > 0) {
+    std::vector<std::string>::const_iterator token = tokens.begin();
+    for (; token + 1 != tokens.end(); ++token) {
+      Tokenize(sourcePhrase, *token, true, true);
+    }
+    // last position. LHS non-term
+    Tokenize(sourcePhrase, *token, false, true);
+  }
+  return sourcePhrase;
+}
+
+const PhraseNode* OnDiskQuery::Query(const SourcePhrase& sourcePhrase)
+{
+  const PhraseNode *node = &m_wrapper.GetRootSourceNode();
+  assert(node);
+
+  for (size_t pos = 0; pos < sourcePhrase.GetSize(); ++pos) {
+    const Word &word = sourcePhrase.GetWord(pos);
+    node = node->GetChild(word, m_wrapper);
+    if (node == NULL) {
+      break;
+    }
+  }
+  return node;
+}
+
+}

tools/mosesdecoder/OnDiskPt/OnDiskQuery.h

@@ -0,0 +1,39 @@
+#pragma once
+#include <string>
+#include <vector>
+#include "OnDiskWrapper.h"
+#include "Phrase.h"
+#include "SourcePhrase.h"
+#include "Word.h"
+#include "PhraseNode.h"
+
+
+namespace OnDiskPt
+{
+
+class OnDiskQuery
+{
+private:
+  OnDiskWrapper &m_wrapper;
+
+public:
+
+  OnDiskQuery(OnDiskWrapper &wrapper):m_wrapper(wrapper) {}
+
+  void Tokenize(Phrase &phrase,
+                const std::string &token,
+                bool addSourceNonTerm,
+                bool addTargetNonTerm);
+
+  SourcePhrase Tokenize(const std::vector<std::string>& tokens);
+
+  const PhraseNode *Query(const SourcePhrase& sourcePhrase);
+
+  inline const PhraseNode *Query(const std::vector<std::string>& tokens) {
+    return Query(Tokenize(tokens));
+  }
+
+};
+
+
+}

tools/mosesdecoder/OnDiskPt/OnDiskWrapper.cpp

@@ -0,0 +1,223 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#ifdef WIN32
+#include <direct.h>
+#endif
+#include <sys/stat.h>
+#include <string>
+#include "OnDiskWrapper.h"
+#include "moses/Util.h"
+#include "util/exception.hh"
+#include "util/string_stream.hh"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+int OnDiskWrapper::VERSION_NUM = 7;
+
+OnDiskWrapper::OnDiskWrapper()
+{
+}
+
+OnDiskWrapper::~OnDiskWrapper()
+{
+  delete m_rootSourceNode;
+}
+
+void OnDiskWrapper::BeginLoad(const std::string &filePath)
+{
+  if (!OpenForLoad(filePath)) {
+    UTIL_THROW(util::FileOpenException, "Couldn't open for loading: " << filePath);
+  }
+
+  if (!m_vocab.Load(*this))
+    UTIL_THROW(util::FileOpenException, "Couldn't load vocab");
+
+  uint64_t rootFilePos = GetMisc("RootNodeOffset");
+  m_rootSourceNode = new PhraseNode(rootFilePos, *this);
+}
+
+bool OnDiskWrapper::OpenForLoad(const std::string &filePath)
+{
+  m_fileSource.open((filePath + "/Source.dat").c_str(), ios::in | ios::binary);
+  UTIL_THROW_IF(!m_fileSource.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Source.dat");
+
+  m_fileTargetInd.open((filePath + "/TargetInd.dat").c_str(), ios::in | ios::binary);
+  UTIL_THROW_IF(!m_fileTargetInd.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/TargetInd.dat");
+
+  m_fileTargetColl.open((filePath + "/TargetColl.dat").c_str(), ios::in | ios::binary);
+  UTIL_THROW_IF(!m_fileTargetColl.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/TargetColl.dat");
+
+  m_fileVocab.open((filePath + "/Vocab.dat").c_str(), ios::in);
+  UTIL_THROW_IF(!m_fileVocab.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Vocab.dat");
+
+  m_fileMisc.open((filePath + "/Misc.dat").c_str(), ios::in);
+  UTIL_THROW_IF(!m_fileMisc.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Misc.dat");
+
+  // set up root node
+  LoadMisc();
+  m_numSourceFactors = GetMisc("NumSourceFactors");
+  m_numTargetFactors = GetMisc("NumTargetFactors");
+  m_numScores = GetMisc("NumScores");
+
+  return true;
+}
+
+bool OnDiskWrapper::LoadMisc()
+{
+  char line[100000];
+
+  while(m_fileMisc.getline(line, 100000)) {
+    vector<string> tokens;
+    Moses::Tokenize(tokens, line);
+    UTIL_THROW_IF2(tokens.size() != 2, "Except key value. Found " << line);
+
+
+    const string &key = tokens[0];
+    m_miscInfo[key] =  Moses::Scan<uint64_t>(tokens[1]);
+  }
+
+  return true;
+}
+
+void OnDiskWrapper::BeginSave(const std::string &filePath
+                              , int numSourceFactors, int	numTargetFactors, int numScores)
+{
+  m_numSourceFactors = numSourceFactors;
+  m_numTargetFactors = numTargetFactors;
+  m_numScores = numScores;
+  m_filePath = filePath;
+
+#ifdef WIN32
+  mkdir(filePath.c_str());
+#else
+  mkdir(filePath.c_str(), 0777);
+#endif
+
+  m_fileSource.open((filePath + "/Source.dat").c_str(), ios::out | ios::in | ios::binary | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileSource.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Source.dat");
+
+  m_fileTargetInd.open((filePath + "/TargetInd.dat").c_str(), ios::out | ios::binary | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileTargetInd.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/TargetInd.dat");
+
+  m_fileTargetColl.open((filePath + "/TargetColl.dat").c_str(), ios::out | ios::binary | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileTargetColl.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/TargetColl.dat");
+
+  m_fileVocab.open((filePath + "/Vocab.dat").c_str(), ios::out | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileVocab.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Vocab.dat");
+
+  m_fileMisc.open((filePath + "/Misc.dat").c_str(), ios::out | ios::ate | ios::trunc);
+  UTIL_THROW_IF(!m_fileMisc.is_open(),
+                util::FileOpenException,
+                "Couldn't open file " << filePath << "/Misc.dat");
+
+  // offset by 1. 0 offset is reserved
+  char c = 0xff;
+  m_fileSource.write(&c, 1);
+  UTIL_THROW_IF2(1 != m_fileSource.tellp(),
+                 "Couldn't write to stream m_fileSource");
+
+  m_fileTargetInd.write(&c, 1);
+  UTIL_THROW_IF2(1 != m_fileTargetInd.tellp(),
+                 "Couldn't write to stream m_fileTargetInd");
+
+  m_fileTargetColl.write(&c, 1);
+  UTIL_THROW_IF2(1 != m_fileTargetColl.tellp(),
+                 "Couldn't write to stream m_fileTargetColl");
+
+  // set up root node
+  UTIL_THROW_IF2(GetNumCounts() != 1,
+                 "Not sure what this is...");
+
+  vector<float> counts(GetNumCounts());
+  counts[0] = DEFAULT_COUNT;
+  m_rootSourceNode = new PhraseNode();
+  m_rootSourceNode->AddCounts(counts);
+}
+
+void OnDiskWrapper::EndSave()
+{
+  bool ret = m_rootSourceNode->Saved();
+  UTIL_THROW_IF2(!ret, "Root node not saved");
+
+  GetVocab().Save(*this);
+
+  SaveMisc();
+
+  m_fileMisc.close();
+  m_fileVocab.close();
+  m_fileSource.close();
+  m_fileTarget.close();
+  m_fileTargetInd.close();
+  m_fileTargetColl.close();
+}
+
+void OnDiskWrapper::SaveMisc()
+{
+  m_fileMisc << "Version " << VERSION_NUM << endl;
+  m_fileMisc << "NumSourceFactors " << m_numSourceFactors << endl;
+  m_fileMisc << "NumTargetFactors " << m_numTargetFactors << endl;
+  m_fileMisc << "NumScores " << m_numScores << endl;
+  m_fileMisc << "RootNodeOffset " << m_rootSourceNode->GetFilePos() << endl;
+}
+
+size_t OnDiskWrapper::GetSourceWordSize() const
+{
+  return sizeof(uint64_t) + sizeof(char);
+}
+
+size_t OnDiskWrapper::GetTargetWordSize() const
+{
+  return sizeof(uint64_t) + sizeof(char);
+}
+
+uint64_t OnDiskWrapper::GetMisc(const std::string &key) const
+{
+  std::map<std::string, uint64_t>::const_iterator iter;
+  iter = m_miscInfo.find(key);
+  UTIL_THROW_IF2(iter == m_miscInfo.end()
+                 , "Couldn't find value for key " << key
+                );
+
+  return iter->second;
+}
+
+
+}

tools/mosesdecoder/OnDiskPt/OnDiskWrapper.h

@@ -0,0 +1,111 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include <fstream>
+#include "Vocab.h"
+#include "PhraseNode.h"
+
+namespace OnDiskPt
+{
+const float DEFAULT_COUNT = 66666;
+
+/** Global class with misc information need to create and use the on-disk rule table.
+ * 1 object of this class should be instantiated per rule table.
+ * Currently only hierarchical/syntax models use this, but can & should be used with pb models too
+ */
+class OnDiskWrapper
+{
+protected:
+  Vocab m_vocab;
+  std::string m_filePath;
+  int m_numSourceFactors, m_numTargetFactors, m_numScores;
+  std::fstream m_fileMisc, m_fileVocab, m_fileSource, m_fileTarget, m_fileTargetInd, m_fileTargetColl;
+
+  size_t m_defaultNodeSize;
+  PhraseNode *m_rootSourceNode;
+
+  std::map<std::string, uint64_t> m_miscInfo;
+
+  void SaveMisc();
+  bool OpenForLoad(const std::string &filePath);
+  bool LoadMisc();
+
+public:
+  static int VERSION_NUM;
+
+  OnDiskWrapper();
+  ~OnDiskWrapper();
+
+  void BeginLoad(const std::string &filePath);
+
+  void BeginSave(const std::string &filePath
+                 , int numSourceFactors, int	numTargetFactors, int numScores);
+  void EndSave();
+
+  Vocab &GetVocab() {
+    return m_vocab;
+  }
+  const Vocab &GetVocab() const {
+    return m_vocab;
+  }
+
+  size_t GetSourceWordSize() const;
+  size_t GetTargetWordSize() const;
+
+  std::fstream &GetFileSource() {
+    return m_fileSource;
+  }
+  std::fstream &GetFileTargetInd() {
+    return m_fileTargetInd;
+  }
+  std::fstream &GetFileTargetColl() {
+    return m_fileTargetColl;
+  }
+  std::fstream &GetFileVocab() {
+    return m_fileVocab;
+  }
+
+  size_t GetNumSourceFactors() const {
+    return m_numSourceFactors;
+  }
+  size_t GetNumTargetFactors() const {
+    return m_numTargetFactors;
+  }
+
+  size_t GetNumScores() const {
+    return m_numScores;
+  }
+  size_t GetNumCounts() const {
+    return 1;
+  }
+
+  PhraseNode &GetRootSourceNode() {
+    return *m_rootSourceNode;
+  }
+  const PhraseNode &GetRootSourceNode() const {
+    return *m_rootSourceNode;
+  }
+
+  uint64_t GetMisc(const std::string &key) const;
+
+};
+
+}

tools/mosesdecoder/OnDiskPt/Phrase.cpp

@@ -0,0 +1,108 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <iostream>
+#include "moses/Util.h"
+#include "Phrase.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+
+void Phrase::AddWord(WordPtr word)
+{
+  m_words.push_back(word);
+}
+
+void Phrase::AddWord(WordPtr word, size_t pos)
+{
+  UTIL_THROW_IF2(!(pos < m_words.size()),
+                 "Trying to get word " << pos << " when phrase size is " << m_words.size());
+  m_words.insert(m_words.begin() + pos + 1, word);
+}
+
+int Phrase::Compare(const Phrase &compare) const
+{
+  int ret = 0;
+  for (size_t pos = 0; pos < GetSize(); ++pos) {
+    if (pos >= compare.GetSize()) {
+      // we're bigger than the other. Put 1st
+      ret = -1;
+      break;
+    }
+
+    const Word &thisWord = GetWord(pos)
+                           ,&compareWord = compare.GetWord(pos);
+    int wordRet = thisWord.Compare(compareWord);
+    if (wordRet != 0) {
+      ret = wordRet;
+      break;
+    }
+  }
+
+  if (ret == 0) {
+    assert(compare.GetSize() >= GetSize());
+    ret = (compare.GetSize() > GetSize()) ? 1 : 0;
+  }
+  return ret;
+}
+
+//! transitive comparison
+bool Phrase::operator<(const Phrase &compare) const
+{
+  int ret = Compare(compare);
+  return ret < 0;
+}
+
+bool Phrase::operator>(const Phrase &compare) const
+{
+  int ret = Compare(compare);
+  return ret > 0;
+}
+
+bool Phrase::operator==(const Phrase &compare) const
+{
+  int ret = Compare(compare);
+  return ret == 0;
+}
+
+void Phrase::DebugPrint(ostream &out, const Vocab &vocab) const
+{
+  for (size_t pos = 0; pos < GetSize(); ++pos) {
+    const Word &word = GetWord(pos);
+    word.DebugPrint(out, vocab);
+    out << " ";
+  }
+}
+
+std::ostream& operator<<(std::ostream &out, const Phrase &phrase)
+{
+  for (size_t pos = 0; pos < phrase.GetSize(); ++pos) {
+    const Word &word = phrase.GetWord(pos);
+    out << word << " ";
+  }
+
+  return out;
+}
+
+}
+

tools/mosesdecoder/OnDiskPt/Phrase.h

@@ -0,0 +1,66 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <vector>
+#include <iostream>
+#include <boost/shared_ptr.hpp>
+#include "Word.h"
+
+namespace OnDiskPt
+{
+class Vocab;
+
+
+/** A contiguous phrase. SourcePhrase & TargetPhrase inherit from this and add the on-disk functionality
+ */
+class Phrase
+{
+  friend std::ostream& operator<<(std::ostream&, const Phrase&);
+
+protected:
+  std::vector<WordPtr>	m_words;
+
+public:
+  Phrase() {
+  }
+
+  virtual ~Phrase() {}
+
+  void AddWord(WordPtr word);
+  void AddWord(WordPtr word, size_t pos);
+
+  const Word &GetWord(size_t pos) const {
+    return *m_words[pos];
+  }
+  size_t GetSize() const {
+    return m_words.size();
+  }
+
+  virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const;
+
+  int Compare(const Phrase &compare) const;
+  bool operator<(const Phrase &compare) const;
+  bool operator>(const Phrase &compare) const;
+  bool operator==(const Phrase &compare) const;
+};
+
+typedef boost::shared_ptr<Phrase> PhrasePtr;
+
+}

tools/mosesdecoder/OnDiskPt/PhraseNode.cpp

@@ -0,0 +1,268 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include "PhraseNode.h"
+#include "OnDiskWrapper.h"
+#include "TargetPhraseCollection.h"
+#include "SourcePhrase.h"
+#include "moses/Util.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+size_t PhraseNode::GetNodeSize(size_t numChildren, size_t wordSize, size_t countSize)
+{
+  size_t ret = sizeof(uint64_t) * 2 // num children, value
+               + (wordSize + sizeof(uint64_t)) * numChildren // word + ptr to next source node
+               + sizeof(float) * countSize; // count info
+  return ret;
+}
+
+PhraseNode::PhraseNode()
+  : m_value(0)
+  ,m_currChild(NULL)
+  ,m_saved(false)
+  ,m_memLoad(NULL)
+{
+}
+
+PhraseNode::PhraseNode(uint64_t filePos, OnDiskWrapper &onDiskWrapper)
+  :m_counts(onDiskWrapper.GetNumCounts())
+{
+  // load saved node
+  m_filePos = filePos;
+
+  size_t countSize = onDiskWrapper.GetNumCounts();
+
+  std::fstream &file = onDiskWrapper.GetFileSource();
+  file.seekg(filePos);
+  assert(filePos == (uint64_t)file.tellg());
+
+  file.read((char*) &m_numChildrenLoad, sizeof(uint64_t));
+
+  size_t memAlloc = GetNodeSize(m_numChildrenLoad, onDiskWrapper.GetSourceWordSize(), countSize);
+  m_memLoad = (char*) malloc(memAlloc);
+
+  // go to start of node again
+  file.seekg(filePos);
+  assert(filePos == (uint64_t)file.tellg());
+
+  // read everything into memory
+  file.read(m_memLoad, memAlloc);
+  assert(filePos + memAlloc == (uint64_t)file.tellg());
+
+  // get value
+  m_value = ((uint64_t*)m_memLoad)[1];
+
+  // get counts
+  float *memFloat = (float*) (m_memLoad + sizeof(uint64_t) * 2);
+
+  assert(countSize == 1);
+  m_counts[0] = memFloat[0];
+
+  m_memLoadLast = m_memLoad + memAlloc;
+}
+
+PhraseNode::~PhraseNode()
+{
+  free(m_memLoad);
+}
+
+float PhraseNode::GetCount(size_t ind) const
+{
+  return m_counts[ind];
+}
+
+void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimit)
+{
+  UTIL_THROW_IF2(m_saved, "Already saved");
+
+  // save this node
+  m_targetPhraseColl.Sort(tableLimit);
+  m_targetPhraseColl.Save(onDiskWrapper);
+  m_value = m_targetPhraseColl.GetFilePos();
+
+  size_t numCounts = onDiskWrapper.GetNumCounts();
+
+  size_t memAlloc = GetNodeSize(GetSize(), onDiskWrapper.GetSourceWordSize(), numCounts);
+  char *mem = (char*) malloc(memAlloc);
+  //memset(mem, 0xfe, memAlloc);
+
+  size_t memUsed = 0;
+  uint64_t *memArray = (uint64_t*) mem;
+  memArray[0] = GetSize(); // num of children
+  memArray[1] = m_value;   // file pos of corresponding target phrases
+  memUsed += 2 * sizeof(uint64_t);
+
+  // count info
+  float *memFloat = (float*) (mem + memUsed);
+  UTIL_THROW_IF2(numCounts != 1, "Can only store 1 phrase count");
+  memFloat[0] = (m_counts.size() == 0) ? DEFAULT_COUNT : m_counts[0]; // if count = 0, put in very large num to make sure its still used. HACK
+  memUsed += sizeof(float) * numCounts;
+
+  // recursively save chm_countsildren
+  ChildColl::iterator iter;
+  for (iter = m_children.begin(); iter != m_children.end(); ++iter) {
+    const Word &childWord = iter->first;
+    PhraseNode &childNode = iter->second;
+
+    // recursive
+    if (!childNode.Saved())
+      childNode.Save(onDiskWrapper, pos + 1, tableLimit);
+
+    char *currMem = mem + memUsed;
+    size_t wordMemUsed = childWord.WriteToMemory(currMem);
+    memUsed += wordMemUsed;
+
+    uint64_t *memArray = (uint64_t*) (mem + memUsed);
+    memArray[0] = childNode.GetFilePos();
+    memUsed += sizeof(uint64_t);
+
+  }
+
+  // save this node
+  //Moses::DebugMem(mem, memAlloc);
+  assert(memUsed == memAlloc);
+
+  std::fstream &file = onDiskWrapper.GetFileSource();
+  m_filePos = file.tellp();
+  file.seekp(0, ios::end);
+  file.write(mem, memUsed);
+
+  uint64_t endPos = file.tellp();
+  assert(m_filePos + memUsed == endPos);
+
+  free(mem);
+
+  m_children.clear();
+  m_saved = true;
+}
+
+void PhraseNode::AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase
+                                 , OnDiskWrapper &onDiskWrapper, size_t tableLimit
+                                 , const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort)
+{
+  AddTargetPhrase(0, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts, spShort);
+}
+
+void PhraseNode::AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase
+                                 , TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper
+                                 , size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort)
+{
+  size_t phraseSize = sourcePhrase.GetSize();
+  if (pos < phraseSize) {
+    const Word &word = sourcePhrase.GetWord(pos);
+
+    PhraseNode &node = m_children[word];
+    if (m_currChild != &node) {
+      // new node
+      node.SetPos(pos);
+
+      if (m_currChild) {
+        m_currChild->Save(onDiskWrapper, pos, tableLimit);
+      }
+
+      m_currChild = &node;
+    }
+
+    // keep searching for target phrase node..
+    node.AddTargetPhrase(pos + 1, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts, spShort);
+  } else {
+    // drilled down to the right node
+    m_counts = counts;
+    targetPhrase->SetSourcePhrase(spShort);
+    m_targetPhraseColl.AddTargetPhrase(targetPhrase);
+  }
+}
+
+const PhraseNode *PhraseNode::GetChild(const Word &wordSought, OnDiskWrapper &onDiskWrapper) const
+{
+  const PhraseNode *ret = NULL;
+
+  int l = 0;
+  int r = m_numChildrenLoad - 1;
+  int x;
+
+  while (r >= l) {
+    x = (l + r) / 2;
+
+    Word wordFound;
+    uint64_t childFilePos;
+    GetChild(wordFound, childFilePos, x, onDiskWrapper);
+
+    if (wordSought == wordFound) {
+      ret = new PhraseNode(childFilePos, onDiskWrapper);
+      break;
+    }
+    if (wordSought < wordFound)
+      r = x - 1;
+    else
+      l = x + 1;
+  }
+
+  return ret;
+}
+
+void PhraseNode::GetChild(Word &wordFound, uint64_t &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const
+{
+
+  size_t wordSize = onDiskWrapper.GetSourceWordSize();
+  size_t childSize = wordSize + sizeof(uint64_t);
+
+  char *currMem = m_memLoad
+                  + sizeof(uint64_t) * 2 // size & file pos of target phrase coll
+                  + sizeof(float) * onDiskWrapper.GetNumCounts() // count info
+                  + childSize * ind;
+
+  size_t memRead = ReadChild(wordFound, childFilePos, currMem);
+  assert(memRead == childSize);
+}
+
+size_t PhraseNode::ReadChild(Word &wordFound, uint64_t &childFilePos, const char *mem) const
+{
+  size_t memRead = wordFound.ReadFromMemory(mem);
+
+  const char *currMem = mem + memRead;
+  uint64_t *memArray = (uint64_t*) (currMem);
+  childFilePos = memArray[0];
+
+  memRead += sizeof(uint64_t);
+  return memRead;
+}
+
+TargetPhraseCollection::shared_ptr
+PhraseNode::
+GetTargetPhraseCollection(size_t tableLimit, OnDiskWrapper &onDiskWrapper) const
+{
+  TargetPhraseCollection::shared_ptr ret(new TargetPhraseCollection);
+  if (m_value > 0) ret->ReadFromFile(tableLimit, m_value, onDiskWrapper);
+  return ret;
+}
+
+std::ostream& operator<<(std::ostream &out, const PhraseNode &node)
+{
+  out << "node (" << node.GetFilePos() << "," << node.GetValue() << "," << node.m_pos << ")";
+  return out;
+}
+
+}
+

tools/mosesdecoder/OnDiskPt/PhraseNode.h

@@ -0,0 +1,108 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <fstream>
+#include <vector>
+#include <map>
+#include "Word.h"
+#include "TargetPhraseCollection.h"
+#include "Phrase.h"
+
+namespace OnDiskPt
+{
+
+class OnDiskWrapper;
+class SourcePhrase;
+
+/** A node in the source tree trie */
+class PhraseNode
+{
+  friend std::ostream& operator<<(std::ostream&, const PhraseNode&);
+protected:
+  uint64_t m_filePos, m_value;
+
+  typedef std::map<Word, PhraseNode> ChildColl;
+  ChildColl m_children;
+  PhraseNode *m_currChild;
+  bool m_saved;
+  size_t m_pos;
+  std::vector<float> m_counts;
+
+  TargetPhraseCollection m_targetPhraseColl;
+
+  char *m_memLoad, *m_memLoadLast;
+  uint64_t m_numChildrenLoad;
+
+  void AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase
+                       , TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper
+                       , size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
+  size_t ReadChild(Word &wordFound, uint64_t &childFilePos, const char *mem) const;
+  void GetChild(Word &wordFound, uint64_t &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const;
+
+public:
+  static size_t GetNodeSize(size_t numChildren, size_t wordSize, size_t countSize);
+
+  PhraseNode(); // unsaved node
+  PhraseNode(uint64_t filePos, OnDiskWrapper &onDiskWrapper); // load saved node
+  ~PhraseNode();
+
+  void Add(const Word &word, uint64_t nextFilePos, size_t wordSize);
+  void Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimit);
+
+  void AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase
+                       , OnDiskWrapper &onDiskWrapper, size_t tableLimit
+                       , const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
+
+  uint64_t GetFilePos() const {
+    return m_filePos;
+  }
+  uint64_t GetValue() const {
+    return m_value;
+  }
+  void SetValue(uint64_t value) {
+    m_value = value;
+  }
+  size_t GetSize() const {
+    return m_children.size();
+  }
+
+  bool Saved() const {
+    return m_saved;
+  }
+
+  void SetPos(size_t pos) {
+    m_pos = pos;
+  }
+
+  const PhraseNode *GetChild(const Word &wordSought, OnDiskWrapper &onDiskWrapper) const;
+
+  TargetPhraseCollection::shared_ptr
+  GetTargetPhraseCollection(size_t tableLimit,
+                            OnDiskWrapper &onDiskWrapper) const;
+
+  void AddCounts(const std::vector<float> &counts) {
+    m_counts = counts;
+  }
+  float GetCount(size_t ind) const;
+
+};
+
+}
+

tools/mosesdecoder/OnDiskPt/SourcePhrase.cpp

@@ -0,0 +1,27 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include "SourcePhrase.h"
+
+namespace OnDiskPt
+{
+
+}
+
+

tools/mosesdecoder/OnDiskPt/SourcePhrase.h

@@ -0,0 +1,38 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <vector>
+#include "Phrase.h"
+#include "Word.h"
+
+namespace OnDiskPt
+{
+
+/** A source phrase. No extension of a norm Phrase class because source phrases are saved as tries.
+ */
+class SourcePhrase: public Phrase
+{
+protected:
+
+public:
+};
+
+
+}

tools/mosesdecoder/OnDiskPt/TargetPhrase.cpp

@@ -0,0 +1,402 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include <iostream>
+#include "moses/Util.h"
+#include "TargetPhrase.h"
+#include "OnDiskWrapper.h"
+#include "util/exception.hh"
+
+#include <boost/algorithm/string.hpp>
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+TargetPhrase::TargetPhrase(size_t numScores)
+  :m_scores(numScores)
+{
+}
+
+TargetPhrase::TargetPhrase(const 	TargetPhrase &copy)
+  :Phrase(copy)
+  ,m_scores(copy.m_scores)
+{
+
+}
+
+TargetPhrase::~TargetPhrase()
+{
+}
+
+void TargetPhrase::SetLHS(WordPtr lhs)
+{
+  AddWord(lhs);
+}
+
+void TargetPhrase::Create1AlignFromString(const std::string &align1Str)
+{
+  vector<size_t> alignPoints;
+  Moses::Tokenize<size_t>(alignPoints, align1Str, "-");
+  UTIL_THROW_IF2(alignPoints.size() != 2, "Incorrectly formatted word alignment: " << align1Str);
+  m_align.push_back(pair<size_t, size_t>(alignPoints[0], alignPoints[1]) );
+}
+
+void TargetPhrase::CreateAlignFromString(const std::string &alignStr)
+{
+  vector<std::string> alignPairs;
+  boost::split(alignPairs, alignStr, boost::is_any_of("\t "));
+  for (size_t i = 0; i < alignPairs.size(); ++i) {
+    vector<size_t> alignPoints;
+    Moses::Tokenize<size_t>(alignPoints, alignPairs[i], "-");
+    m_align.push_back(pair<size_t, size_t>(alignPoints[0], alignPoints[1]) );
+  }
+}
+
+
+void TargetPhrase::SetScore(float score, size_t ind)
+{
+  assert(ind < m_scores.size());
+  m_scores[ind] = score;
+}
+
+class AlignOrderer
+{
+public:
+  bool operator()(const AlignPair &a, const AlignPair &b) const {
+    return a.first < b.first;
+  }
+};
+
+void TargetPhrase::SortAlign()
+{
+  std::sort(m_align.begin(), m_align.end(), AlignOrderer());
+}
+
+char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const
+{
+  size_t phraseSize = GetSize();
+  size_t targetWordSize = onDiskWrapper.GetTargetWordSize();
+
+  const PhrasePtr sp = GetSourcePhrase();
+  size_t spSize = sp->GetSize();
+  size_t sourceWordSize = onDiskWrapper.GetSourceWordSize();
+
+  size_t memNeeded = sizeof(uint64_t)						// num of words
+                     + targetWordSize * phraseSize	// actual words. lhs as last words
+                     + sizeof(uint64_t)					// num source words
+                     + sourceWordSize * spSize;   // actual source words
+
+  memUsed = 0;
+  uint64_t *mem = (uint64_t*) malloc(memNeeded);
+
+  // write size
+  mem[0] = phraseSize;
+  memUsed += sizeof(uint64_t);
+
+  // write each word
+  for (size_t pos = 0; pos < phraseSize; ++pos) {
+    const Word &word = GetWord(pos);
+    char *currPtr = (char*)mem + memUsed;
+    memUsed += word.WriteToMemory((char*) currPtr);
+  }
+
+  // write size of source phrase and all source words
+  char *currPtr = (char*)mem + memUsed;
+  uint64_t *memTmp = (uint64_t*) currPtr;
+  memTmp[0] = spSize;
+  memUsed += sizeof(uint64_t);
+  for (size_t pos = 0; pos < spSize; ++pos) {
+    const Word &word = sp->GetWord(pos);
+    char *currPtr = (char*)mem + memUsed;
+    memUsed += word.WriteToMemory((char*) currPtr);
+  }
+
+  assert(memUsed == memNeeded);
+  return (char *) mem;
+}
+
+void TargetPhrase::Save(OnDiskWrapper &onDiskWrapper)
+{
+  // save in target ind
+  size_t memUsed;
+  char *mem = WriteToMemory(onDiskWrapper, memUsed);
+
+  std::fstream &file = onDiskWrapper.GetFileTargetInd();
+
+  uint64_t startPos = file.tellp();
+
+  file.seekp(0, ios::end);
+  file.write(mem, memUsed);
+
+#ifndef NDEBUG
+  uint64_t endPos = file.tellp();
+  assert(startPos + memUsed == endPos);
+#endif
+
+  m_filePos = startPos;
+  free(mem);
+}
+
+char *TargetPhrase::WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const
+{
+  // allocate mem
+  size_t numScores = onDiskWrapper.GetNumScores()
+                     ,numAlign = GetAlign().size();
+  size_t sparseFeatureSize = m_sparseFeatures.size();
+  size_t propSize = m_property.size();
+
+  size_t memNeeded = sizeof(uint64_t) // file pos (phrase id)
+                     + sizeof(uint64_t) + 2 * sizeof(uint64_t) * numAlign // align
+                     + sizeof(float) * numScores // scores
+                     + sizeof(uint64_t) + sparseFeatureSize // sparse features string
+                     + sizeof(uint64_t) + propSize; // property string
+
+  char *mem = (char*) malloc(memNeeded);
+  //memset(mem, 0, memNeeded);
+
+  memUsed = 0;
+
+  // phrase id
+  memcpy(mem, &m_filePos, sizeof(uint64_t));
+  memUsed += sizeof(uint64_t);
+
+  // align
+  size_t tmp = WriteAlignToMemory(mem + memUsed);
+  memUsed += tmp;
+
+  // scores
+  memUsed += WriteScoresToMemory(mem + memUsed);
+
+  // sparse features
+  memUsed += WriteStringToMemory(mem + memUsed, m_sparseFeatures);
+
+  // property string
+  memUsed += WriteStringToMemory(mem + memUsed, m_property);
+
+  //DebugMem(mem, memNeeded);
+  assert(memNeeded == memUsed);
+  return mem;
+}
+
+size_t TargetPhrase::WriteStringToMemory(char *mem, const std::string &str) const
+{
+  size_t memUsed = 0;
+  uint64_t *memTmp = (uint64_t*) mem;
+
+  size_t strSize = str.size();
+  memTmp[0] = strSize;
+  memUsed += sizeof(uint64_t);
+
+  const char *charStr = str.c_str();
+  memcpy(mem + memUsed, charStr, strSize);
+  memUsed += strSize;
+
+  return memUsed;
+}
+
+size_t TargetPhrase::WriteAlignToMemory(char *mem) const
+{
+  size_t memUsed = 0;
+
+  // num of alignments
+  uint64_t numAlign = m_align.size();
+  memcpy(mem, &numAlign, sizeof(numAlign));
+  memUsed += sizeof(numAlign);
+
+  // actual alignments
+  AlignType::const_iterator iter;
+  for (iter = m_align.begin(); iter != m_align.end(); ++iter) {
+    const AlignPair &alignPair = *iter;
+
+    memcpy(mem + memUsed, &alignPair.first, sizeof(alignPair.first));
+    memUsed += sizeof(alignPair.first);
+
+    memcpy(mem + memUsed, &alignPair.second, sizeof(alignPair.second));
+    memUsed += sizeof(alignPair.second);
+  }
+
+  return memUsed;
+}
+
+size_t TargetPhrase::WriteScoresToMemory(char *mem) const
+{
+  float *scoreMem = (float*) mem;
+
+  for (size_t ind = 0; ind < m_scores.size(); ++ind)
+    scoreMem[ind] = m_scores[ind];
+
+  size_t memUsed = sizeof(float) * m_scores.size();
+  return memUsed;
+}
+
+uint64_t TargetPhrase::ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl)
+{
+  assert(filePos == (uint64_t)fileTPColl.tellg());
+
+  uint64_t memUsed = 0;
+  fileTPColl.read((char*) &m_filePos, sizeof(uint64_t));
+  memUsed += sizeof(uint64_t);
+  assert(m_filePos != 0);
+
+  memUsed += ReadAlignFromFile(fileTPColl);
+  assert((memUsed + filePos) == (uint64_t)fileTPColl.tellg());
+
+  memUsed += ReadScoresFromFile(fileTPColl);
+  assert((memUsed + filePos) == (uint64_t)fileTPColl.tellg());
+
+  // sparse features
+  memUsed += ReadStringFromFile(fileTPColl, m_sparseFeatures);
+
+  // properties
+  memUsed += ReadStringFromFile(fileTPColl, m_property);
+
+  return memUsed;
+}
+
+uint64_t TargetPhrase::ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr)
+{
+  uint64_t bytesRead = 0;
+
+  uint64_t strSize;
+  fileTPColl.read((char*) &strSize, sizeof(uint64_t));
+  bytesRead += sizeof(uint64_t);
+
+  if (strSize) {
+    char *mem = (char*) malloc(strSize + 1);
+    mem[strSize] = '\0';
+    fileTPColl.read(mem, strSize);
+    outStr = string(mem);
+    free(mem);
+
+    bytesRead += strSize;
+  }
+
+  return bytesRead;
+}
+
+uint64_t TargetPhrase::ReadFromFile(std::fstream &fileTP)
+{
+  uint64_t bytesRead = 0;
+
+  fileTP.seekg(m_filePos);
+
+  uint64_t numWords;
+  fileTP.read((char*) &numWords, sizeof(uint64_t));
+  bytesRead += sizeof(uint64_t);
+
+  for (size_t ind = 0; ind < numWords; ++ind) {
+    WordPtr word(new Word());
+    bytesRead += word->ReadFromFile(fileTP);
+    AddWord(word);
+  }
+
+  // read source words
+  uint64_t numSourceWords;
+  fileTP.read((char*) &numSourceWords, sizeof(uint64_t));
+  bytesRead += sizeof(uint64_t);
+
+  PhrasePtr sp(new SourcePhrase());
+  for (size_t ind = 0; ind < numSourceWords; ++ind) {
+    WordPtr word( new Word());
+    bytesRead += word->ReadFromFile(fileTP);
+    sp->AddWord(word);
+  }
+  SetSourcePhrase(sp);
+
+  return bytesRead;
+}
+
+uint64_t TargetPhrase::ReadAlignFromFile(std::fstream &fileTPColl)
+{
+  uint64_t bytesRead = 0;
+
+  uint64_t numAlign;
+  fileTPColl.read((char*) &numAlign, sizeof(uint64_t));
+  bytesRead += sizeof(uint64_t);
+
+  for (size_t ind = 0; ind < numAlign; ++ind) {
+    AlignPair alignPair;
+    fileTPColl.read((char*) &alignPair.first, sizeof(uint64_t));
+    fileTPColl.read((char*) &alignPair.second, sizeof(uint64_t));
+    m_align.push_back(alignPair);
+
+    bytesRead += sizeof(uint64_t) * 2;
+  }
+
+  return bytesRead;
+}
+
+uint64_t TargetPhrase::ReadScoresFromFile(std::fstream &fileTPColl)
+{
+  UTIL_THROW_IF2(m_scores.size() == 0, "Translation rules must must have some scores");
+
+  uint64_t bytesRead = 0;
+
+  for (size_t ind = 0; ind < m_scores.size(); ++ind) {
+    fileTPColl.read((char*) &m_scores[ind], sizeof(float));
+
+    bytesRead += sizeof(float);
+  }
+
+  std::transform(m_scores.begin(),m_scores.end(),m_scores.begin(), Moses::TransformScore);
+  std::transform(m_scores.begin(),m_scores.end(),m_scores.begin(), Moses::FloorScore);
+
+  return bytesRead;
+}
+
+void TargetPhrase::DebugPrint(ostream &out, const Vocab &vocab) const
+{
+  Phrase::DebugPrint(out, vocab);
+
+  for (size_t ind = 0; ind < m_align.size(); ++ind) {
+    const AlignPair &alignPair = m_align[ind];
+    out << alignPair.first << "-" << alignPair.second << " ";
+  }
+  out << ", ";
+
+  for (size_t ind = 0; ind < m_scores.size(); ++ind) {
+    out << m_scores[ind] << " ";
+  }
+
+  return;
+}
+
+std::ostream& operator<<(std::ostream &out, const TargetPhrase &phrase)
+{
+  out << (const Phrase&) phrase << ", " ;
+
+  for (size_t ind = 0; ind < phrase.m_align.size(); ++ind) {
+    const AlignPair &alignPair = phrase.m_align[ind];
+    out << alignPair.first << "-" << alignPair.second << " ";
+  }
+  out << ", ";
+
+  for (size_t ind = 0; ind < phrase.m_scores.size(); ++ind) {
+    out << phrase.m_scores[ind] << " ";
+  }
+
+  return out;
+}
+
+} // namespace
+

tools/mosesdecoder/OnDiskPt/TargetPhrase.h

@@ -0,0 +1,127 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <fstream>
+#include <string>
+#include <vector>
+#include "Word.h"
+#include "Phrase.h"
+#include "SourcePhrase.h"
+
+namespace Moses
+{
+class PhraseDictionary;
+class TargetPhrase;
+class Phrase;
+}
+
+namespace OnDiskPt
+{
+
+typedef std::pair<uint64_t, uint64_t>  AlignPair;
+typedef std::vector<AlignPair> AlignType;
+
+class Vocab;
+
+/** A target phrase, with the score breakdowns, alignment info and assorted other information it need.
+ *  Readable and writeable to disk
+ */
+class TargetPhrase: public Phrase
+{
+  friend std::ostream& operator<<(std::ostream&, const TargetPhrase&);
+protected:
+  AlignType m_align;
+  PhrasePtr m_sourcePhrase;
+  std::string m_sparseFeatures, m_property;
+
+  std::vector<float> m_scores;
+  uint64_t m_filePos;
+
+  size_t WriteAlignToMemory(char *mem) const;
+  size_t WriteScoresToMemory(char *mem) const;
+  size_t WriteStringToMemory(char *mem, const std::string &str) const;
+
+  uint64_t ReadAlignFromFile(std::fstream &fileTPColl);
+  uint64_t ReadScoresFromFile(std::fstream &fileTPColl);
+  uint64_t ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr);
+
+public:
+  TargetPhrase() {
+  }
+  TargetPhrase(size_t numScores);
+  TargetPhrase(const TargetPhrase &copy);
+  virtual ~TargetPhrase();
+
+  void SetSourcePhrase(PhrasePtr p) {
+    m_sourcePhrase = p;
+  }
+  const PhrasePtr GetSourcePhrase() const {
+    return m_sourcePhrase;
+  }
+  const std::vector<float> &GetScores() const {
+    return m_scores;
+  }
+
+  void SetLHS(WordPtr lhs);
+
+  void Create1AlignFromString(const std::string &align1Str);
+  void CreateAlignFromString(const std::string &align1Str);
+  void SetScore(float score, size_t ind);
+
+  const AlignType &GetAlign() const {
+    return m_align;
+  }
+  void SortAlign();
+
+  char *WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const;
+  char *WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const;
+  void Save(OnDiskWrapper &onDiskWrapper);
+
+  uint64_t GetFilePos() const {
+    return m_filePos;
+  }
+  float GetScore(size_t ind) const {
+    return m_scores[ind];
+  }
+
+  uint64_t ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl);
+  uint64_t ReadFromFile(std::fstream &fileTP);
+
+  virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const;
+
+  const std::string &GetProperty() const {
+    return m_property;
+  }
+
+  void SetProperty(const std::string &value) {
+    m_property = value;
+  }
+
+  const std::string &GetSparseFeatures() const {
+    return m_sparseFeatures;
+  }
+
+  void SetSparseFeatures(const std::string &value) {
+    m_sparseFeatures = value;
+  }
+};
+
+}

tools/mosesdecoder/OnDiskPt/TargetPhraseCollection.cpp

@@ -0,0 +1,171 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <algorithm>
+#include <iostream>
+#include "moses/Util.h"
+#include "TargetPhraseCollection.h"
+#include "Vocab.h"
+#include "OnDiskWrapper.h"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+size_t TargetPhraseCollection::s_sortScoreInd;
+
+TargetPhraseCollection::TargetPhraseCollection()
+  :m_filePos(777)
+{}
+
+TargetPhraseCollection::TargetPhraseCollection(const TargetPhraseCollection &copy)
+  :m_filePos(copy.m_filePos)
+  ,m_debugStr(copy.m_debugStr)
+{
+}
+
+TargetPhraseCollection::~TargetPhraseCollection()
+{
+  Moses::RemoveAllInColl(m_coll);
+}
+
+void TargetPhraseCollection::AddTargetPhrase(TargetPhrase *targetPhrase)
+{
+  m_coll.push_back(targetPhrase);
+}
+
+void TargetPhraseCollection::Sort(size_t tableLimit)
+{
+  std::sort(m_coll.begin(), m_coll.end(), TargetPhraseOrderByScore());
+
+  if (tableLimit && m_coll.size() > tableLimit) {
+    CollType::iterator iter;
+    for (iter = m_coll.begin() + tableLimit ; iter != m_coll.end(); ++iter) {
+      delete *iter;
+    }
+    m_coll.resize(tableLimit);
+  }
+}
+
+void TargetPhraseCollection::Save(OnDiskWrapper &onDiskWrapper)
+{
+  std::fstream &file = onDiskWrapper.GetFileTargetColl();
+
+  size_t memUsed = sizeof(uint64_t);
+  char *mem = (char*) malloc(memUsed);
+
+  // size of coll
+  uint64_t numPhrases = GetSize();
+  ((uint64_t*)mem)[0] = numPhrases;
+
+  // MAIN LOOP
+  CollType::iterator iter;
+  for (iter = m_coll.begin(); iter != m_coll.end(); ++iter) {
+    // save phrase
+    TargetPhrase &targetPhrase = **iter;
+    targetPhrase.Save(onDiskWrapper);
+
+    // save coll
+    size_t memUsedTPOtherInfo;
+    char *memTPOtherInfo = targetPhrase.WriteOtherInfoToMemory(onDiskWrapper, memUsedTPOtherInfo);
+
+    // expand existing mem
+    mem = (char*) realloc(mem, memUsed + memUsedTPOtherInfo);
+    memcpy(mem + memUsed, memTPOtherInfo, memUsedTPOtherInfo);
+    memUsed += memUsedTPOtherInfo;
+
+    free(memTPOtherInfo);
+  }
+
+  // total number of bytes
+  //((uint64_t*)mem)[0] = (uint64_t) memUsed;
+
+  uint64_t startPos = file.tellp();
+  file.seekp(0, ios::end);
+  file.write((char*) mem, memUsed);
+
+  free(mem);
+
+#ifndef NDEBUG
+  uint64_t endPos = file.tellp();
+  assert(startPos + memUsed == endPos);
+#endif
+  m_filePos = startPos;
+
+}
+
+void TargetPhraseCollection::ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper)
+{
+  fstream &fileTPColl = onDiskWrapper.GetFileTargetColl();
+  fstream &fileTP = onDiskWrapper.GetFileTargetInd();
+
+  size_t numScores = onDiskWrapper.GetNumScores();
+
+
+  uint64_t numPhrases;
+
+  uint64_t currFilePos = filePos;
+  fileTPColl.seekg(filePos);
+  fileTPColl.read((char*) &numPhrases, sizeof(uint64_t));
+
+  // table limit
+  if (tableLimit) {
+    numPhrases = std::min(numPhrases, (uint64_t) tableLimit);
+  }
+
+  currFilePos += sizeof(uint64_t);
+
+  for (size_t ind = 0; ind < numPhrases; ++ind) {
+    TargetPhrase *tp = new TargetPhrase(numScores);
+
+    uint64_t sizeOtherInfo = tp->ReadOtherInfoFromFile(currFilePos, fileTPColl);
+    tp->ReadFromFile(fileTP);
+
+    currFilePos += sizeOtherInfo;
+
+    m_coll.push_back(tp);
+  }
+}
+
+uint64_t TargetPhraseCollection::GetFilePos() const
+{
+  return m_filePos;
+}
+
+const std::string TargetPhraseCollection::GetDebugStr() const
+{
+  return m_debugStr;
+}
+
+void TargetPhraseCollection::SetDebugStr(const std::string &str)
+{
+  m_debugStr = str;
+}
+
+const TargetPhrase &TargetPhraseCollection::GetTargetPhrase(size_t ind) const
+{
+  assert(ind < GetSize());
+  return *m_coll[ind];
+}
+
+}
+
+

tools/mosesdecoder/OnDiskPt/TargetPhraseCollection.h

@@ -0,0 +1,84 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#pragma once
+
+#include "TargetPhrase.h"
+#include "Vocab.h"
+#include <boost/shared_ptr.hpp>
+
+namespace Moses
+{
+class TargetPhraseCollection;
+class PhraseDictionary;
+}
+
+namespace OnDiskPt
+{
+
+/** A vector of target phrases
+ */
+class TargetPhraseCollection
+{
+  class TargetPhraseOrderByScore
+  {
+  public:
+    bool operator()(const TargetPhrase* a, const TargetPhrase *b) const {
+      return a->GetScore(s_sortScoreInd) > b->GetScore(s_sortScoreInd);
+    }
+  };
+
+protected:
+  typedef std::vector<TargetPhrase*> CollType;
+  CollType m_coll;
+  uint64_t m_filePos;
+  std::string m_debugStr;
+
+public:
+  typedef boost::shared_ptr<TargetPhraseCollection const> shared_const_ptr;
+  typedef boost::shared_ptr<TargetPhraseCollection> shared_ptr;
+
+  static size_t s_sortScoreInd;
+
+  TargetPhraseCollection();
+  TargetPhraseCollection(const TargetPhraseCollection &copy);
+
+  ~TargetPhraseCollection();
+  void AddTargetPhrase(TargetPhrase *targetPhrase);
+  void Sort(size_t tableLimit);
+
+  void Save(OnDiskWrapper &onDiskWrapper);
+
+  size_t GetSize() const {
+    return m_coll.size();
+  }
+
+  const TargetPhrase &GetTargetPhrase(size_t ind) const;
+
+  uint64_t GetFilePos() const;
+
+  void ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper);
+
+  const std::string GetDebugStr() const;
+  void SetDebugStr(const std::string &str);
+
+};
+
+}
+

tools/mosesdecoder/OnDiskPt/Vocab.cpp

@@ -0,0 +1,101 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include <fstream>
+#include "OnDiskWrapper.h"
+#include "Vocab.h"
+#include "moses/Util.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace OnDiskPt
+{
+
+bool Vocab::Load(OnDiskWrapper &onDiskWrapper)
+{
+  fstream &file = onDiskWrapper.GetFileVocab();
+
+  string line;
+  while(getline(file, line)) {
+    vector<string> tokens;
+    Moses::Tokenize(tokens, line);
+    UTIL_THROW_IF2(tokens.size() != 2, "Vocab file corrupted");
+    const string &key = tokens[0];
+    m_vocabColl[key] =  Moses::Scan<uint64_t>(tokens[1]);
+  }
+
+  // create lookup
+  // assume contiguous vocab id
+  m_lookup.resize(m_vocabColl.size() + 1);
+  m_nextId = m_lookup.size();
+
+  CollType::const_iterator iter;
+  for (iter = m_vocabColl.begin(); iter != m_vocabColl.end(); ++iter) {
+    uint32_t vocabId = iter->second;
+    const std::string &word = iter->first;
+
+    m_lookup[vocabId] = word;
+  }
+
+  return true;
+}
+
+void Vocab::Save(OnDiskWrapper &onDiskWrapper)
+{
+  fstream &file = onDiskWrapper.GetFileVocab();
+  CollType::const_iterator iterVocab;
+  for (iterVocab = m_vocabColl.begin(); iterVocab != m_vocabColl.end(); ++iterVocab) {
+    const string &word = iterVocab->first;
+    uint32_t vocabId = iterVocab->second;
+
+    file << word << " " << vocabId << endl;
+  }
+}
+
+uint64_t Vocab::AddVocabId(const std::string &str)
+{
+  // find string id
+  CollType::const_iterator iter = m_vocabColl.find(str);
+  if (iter == m_vocabColl.end()) {
+    // add new vocab entry
+    m_vocabColl[str] = m_nextId;
+    return m_nextId++;
+  } else {
+    // return existing entry
+    return iter->second;
+  }
+}
+
+uint64_t Vocab::GetVocabId(const std::string &str, bool &found) const
+{
+  // find string id
+  CollType::const_iterator iter = m_vocabColl.find(str);
+  if (iter == m_vocabColl.end()) {
+    found = false;
+    return 0; //return whatever
+  } else {
+    // return existing entry
+    found = true;
+    return iter->second;
+  }
+}
+
+}

tools/mosesdecoder/OnDiskPt/Vocab.h

@@ -0,0 +1,58 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include <map>
+#include "moses/TypeDef.h"
+
+
+namespace OnDiskPt
+{
+
+class OnDiskWrapper;
+
+/* A bidirectional map of string<->contiguous id
+ * No distinction between source and target language
+ */
+class Vocab
+{
+protected:
+  typedef std::map<std::string, uint64_t> CollType;
+  CollType m_vocabColl;
+
+  std::vector<std::string> m_lookup; // opposite of m_vocabColl
+  uint64_t m_nextId; // starts @ 1
+
+public:
+  Vocab()
+    :m_nextId(1) {
+  }
+  uint64_t AddVocabId(const std::string &str);
+  uint64_t GetVocabId(const std::string &str, bool &found) const;
+  const std::string &GetString(uint64_t vocabId) const {
+    return m_lookup[vocabId];
+  }
+
+  bool Load(OnDiskWrapper &onDiskWrapper);
+  void Save(OnDiskWrapper &onDiskWrapper);
+};
+
+}
+

tools/mosesdecoder/OnDiskPt/Word.cpp

@@ -0,0 +1,144 @@
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#include <boost/algorithm/string/predicate.hpp>
+#include "moses/Util.h"
+#include "Word.h"
+
+#include "util/tokenize_piece.hh"
+#include "util/exception.hh"
+
+using namespace std;
+using namespace boost::algorithm;
+
+namespace OnDiskPt
+{
+
+Word::Word(const Word &copy)
+  :m_isNonTerminal(copy.m_isNonTerminal)
+  ,m_vocabId(copy.m_vocabId)
+{}
+
+Word::~Word()
+{}
+
+void Word::CreateFromString(const std::string &inString, Vocab &vocab)
+{
+  if (starts_with(inString, "[") && ends_with(inString, "]")) {
+    // non-term
+    m_isNonTerminal = true;
+    string str = inString.substr(1, inString.size() - 2);
+    m_vocabId = vocab.AddVocabId(str);
+  } else {
+    m_isNonTerminal = false;
+    m_vocabId = vocab.AddVocabId(inString);
+  }
+
+}
+
+size_t Word::WriteToMemory(char *mem) const
+{
+  uint64_t *vocabMem = (uint64_t*) mem;
+  vocabMem[0] = m_vocabId;
+
+  size_t size = sizeof(uint64_t);
+
+  // is non-term
+  char bNonTerm = (char) m_isNonTerminal;
+  mem[size] = bNonTerm;
+  ++size;
+
+  return size;
+}
+
+size_t Word::ReadFromMemory(const char *mem)
+{
+  uint64_t *vocabMem = (uint64_t*) mem;
+  m_vocabId = vocabMem[0];
+
+  size_t memUsed = sizeof(uint64_t);
+
+  // is non-term
+  char bNonTerm;
+  bNonTerm = mem[memUsed];
+  m_isNonTerminal = (bool) bNonTerm;
+  ++memUsed;
+
+  return memUsed;
+}
+
+size_t Word::ReadFromFile(std::fstream &file)
+{
+  const size_t memAlloc = sizeof(uint64_t) + sizeof(char);
+  char mem[sizeof(uint64_t) + sizeof(char)];
+  file.read(mem, memAlloc);
+
+  size_t memUsed = ReadFromMemory(mem);
+  assert(memAlloc == memUsed);
+
+  return memAlloc;
+}
+
+int Word::Compare(const Word &compare) const
+{
+  int ret;
+
+  if (m_isNonTerminal != compare.m_isNonTerminal)
+    return m_isNonTerminal ?-1 : 1;
+
+  if (m_vocabId < compare.m_vocabId)
+    ret = -1;
+  else if (m_vocabId > compare.m_vocabId)
+    ret = 1;
+  else
+    ret = 0;
+
+  return ret;
+}
+
+bool Word::operator<(const Word &compare) const
+{
+  int ret = Compare(compare);
+  return ret < 0;
+}
+
+bool Word::operator==(const Word &compare) const
+{
+  int ret = Compare(compare);
+  return ret == 0;
+}
+
+void Word::DebugPrint(ostream &out, const Vocab &vocab) const
+{
+  const string &str = vocab.GetString(m_vocabId);
+  out << str;
+}
+
+std::ostream& operator<<(std::ostream &out, const Word &word)
+{
+  out << "(";
+  out << word.m_vocabId;
+
+  out << (word.m_isNonTerminal ? "n" : "t");
+  out << ")";
+
+  return out;
+}
+}

tools/mosesdecoder/OnDiskPt/Word.h

@@ -0,0 +1,91 @@
+#pragma once
+// $Id$
+/***********************************************************************
+ Moses - factored phrase-based, hierarchical and syntactic language decoder
+ Copyright (C) 2009 Hieu Hoang
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#include <string>
+#include <vector>
+#include <iostream>
+#include <fstream>
+#include <boost/shared_ptr.hpp>
+#include "Vocab.h"
+
+namespace Moses
+{
+class Word;
+}
+
+namespace OnDiskPt
+{
+class Vocab;
+
+/* A wrapper around a vocab id, and a boolean indicating whther it is a term or non-term.
+ * Factors can be represented by using a vocab string with | character, eg go|VB
+ */
+class Word
+{
+  friend std::ostream& operator<<(std::ostream&, const Word&);
+
+private:
+  bool m_isNonTerminal;
+  uint64_t m_vocabId;
+
+public:
+  explicit Word() {
+  }
+
+  explicit Word(bool isNonTerminal)
+    :m_isNonTerminal(isNonTerminal)
+    ,m_vocabId(0) {
+  }
+
+  Word(const Word &copy);
+  ~Word();
+
+
+  void CreateFromString(const std::string &inString, Vocab &vocab);
+  bool IsNonTerminal() const {
+    return m_isNonTerminal;
+  }
+
+  size_t WriteToMemory(char *mem) const;
+  size_t ReadFromMemory(const char *mem);
+  size_t ReadFromFile(std::fstream &file);
+
+  uint64_t GetVocabId() const {
+    return m_vocabId;
+  }
+
+  void SetVocabId(uint64_t vocabId) {
+    m_vocabId = vocabId;
+  }
+
+  void DebugPrint(std::ostream &out, const Vocab &vocab) const;
+  inline const std::string &GetString(const Vocab &vocab) const {
+    return vocab.GetString(m_vocabId);
+  }
+
+  int Compare(const Word &compare) const;
+  bool operator<(const Word &compare) const;
+  bool operator==(const Word &compare) const;
+
+};
+
+typedef boost::shared_ptr<Word> WordPtr;
+}
+

tools/mosesdecoder/OnDiskPt/queryOnDiskPt.cpp

@@ -0,0 +1,86 @@
+// Query binary phrase tables.
+// Christian Hardmeier, 16 May 2010
+
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#include "moses/Util.h"
+#include "OnDiskWrapper.h"
+#include "SourcePhrase.h"
+#include "OnDiskQuery.h"
+
+using namespace std;
+using namespace OnDiskPt;
+
+void usage();
+
+typedef unsigned int uint;
+
+int main(int argc, char **argv)
+{
+  int tableLimit = 20;
+  std::string ttable = "";
+  // bool useAlignments = false;
+
+  for(int i = 1; i < argc; i++) {
+    if(!strcmp(argv[i], "-tlimit")) {
+      if(i + 1 == argc)
+        usage();
+      tableLimit = atoi(argv[++i]);
+    } else if(!strcmp(argv[i], "-t")) {
+      if(i + 1 == argc)
+        usage();
+      ttable = argv[++i];
+    } else
+      usage();
+  }
+
+  if(ttable == "")
+    usage();
+
+  OnDiskWrapper onDiskWrapper;
+  onDiskWrapper.BeginLoad(ttable);
+  OnDiskQuery onDiskQuery(onDiskWrapper);
+
+  cerr << "Ready..." << endl;
+
+  std::string line;
+  while(getline(std::cin, line)) {
+    std::vector<std::string> tokens;
+    tokens = Moses::Tokenize(line, " ");
+
+    cerr << "line: " << line << endl;
+    const PhraseNode* node = onDiskQuery.Query(tokens);
+
+    if (node) {
+      // source phrase points to a bunch of rules
+      TargetPhraseCollection::shared_ptr coll = node->GetTargetPhraseCollection(tableLimit, onDiskWrapper);
+      string str = coll->GetDebugStr();
+      cout << "Found " << coll->GetSize() << endl;
+
+      for (size_t ind = 0; ind < coll->GetSize(); ++ind) {
+        const TargetPhrase &targetPhrase = coll->GetTargetPhrase(ind);
+        cerr << "  ";
+        targetPhrase.DebugPrint(cerr, onDiskWrapper.GetVocab());
+        cerr << endl;
+      }
+    } else {
+      cout << "Not found" << endl;
+    }
+
+    std::cout << '\n';
+    std::cout.flush();
+  }
+
+  cerr << "Finished." << endl;
+}
+
+void usage()
+{
+  std::cerr << "Usage: queryOnDiskPt [-n <nscores>] [-a] -t <ttable>\n"
+            "-tlimit <table limit>      max number of rules per source phrase (default: 20)\n"
+            "-t <ttable>       phrase table\n";
+  exit(1);
+}

tools/mosesdecoder/README

@@ -0,0 +1,16 @@
+Instructions for building and installing Moses are online:
+   http://www.statmt.org/moses/?n=Development.GetStarted
+If you have g++ and Boost installed, and you want the default compilation with most of things you need, then run:
+   ./bjam -j4
+
+Questions should be directed to the mailing list (don't forget to register before sending emails):
+   http://mailman.mit.edu/mailman/listinfo/moses-support
+   https://github.com/moses-smt/mosesdecoder/compare/master...hieu2
+
+Some of the code is not originally part of Moses, but is periodically copied
+into the source tree from elsewhere:
+
+ * "bjam-files" is taken from Boost.
+ * "util" and "lm" are taken from KenLM: https://github.com/kpu/kenlm
+
+

tools/mosesdecoder/azure-pipelines.yml

@@ -0,0 +1,100 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger:
+- master
+
+pool:
+  #vmImage: 'ubuntu-latest'
+  vmImage: 'ubuntu-16.04'
+
+steps:
+
+- script: |
+    echo Printing some environment information
+    echo HOME: $HOME
+    echo
+    echo UBUNTU VERSION: 
+    cat /etc/lsb-release
+    echo
+    echo CPU INFO
+    cat /proc/cpuinfo
+    echo 
+    echo MEM INFO
+    cat /proc/meminfo
+    echo
+    echo DISK INFO
+    df -h
+    echo 
+    echo PWD: $PWD
+    echo
+    ls
+  displayName: 'Printing some environment information'
+
+
+## Installation commands for Ubuntu 
+- script: |
+    sudo apt-get install  \
+      g++  \
+      git  \
+      subversion \
+      automake \
+      libtool \
+      zlib1g-dev \
+      libicu-dev \
+      libboost-all-dev \
+      libssl-dev \
+      libbz2-dev \
+      liblzma-dev \
+      python-dev \
+      graphviz \
+      imagemagick \
+      make \
+      cmake \
+      libgoogle-perftools-dev \
+      autoconf \
+      doxygen 
+  displayName: 'Install Ubuntu packages'
+
+- script: |   
+    wget "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download"
+    mv download  cmph-2.0.2.tar.gz  
+    tar xvzf cmph-2.0.2.tar.gz
+    cd cmph-2.0.2
+    ./configure --prefix=$PWD
+    make
+    make install
+    cd ..
+  displayName: 'Build and Install cmph'    
+
+- script: |   
+    wget "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download"
+    mv download  xmlrpc-c-1.51.06.tgz
+    tar xvzf xmlrpc-c-1.51.06.tgz
+    cd xmlrpc-c-1.51.06
+    ./configure --prefix=$PWD
+    make
+    make install
+    sudo ldconfig
+    cd ..
+  displayName: 'Build and Install xmlrpc-c'    
+
+- script: |   
+    ./bjam \
+      --with-cmph=$PWD/cmph-2.0.2 \
+      --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \
+      -j2
+  displayName: 'Build Moses'    
+
+# - script: |   
+#     ./bjam \
+#       -j2
+#   displayName: 'Build Moses'  
+
+# - task: ComponentGovernanceComponentDetection@0
+#   inputs:
+#     scanType: 'Register'
+#     verbosity: 'Verbose'
+#     alertWarningLevel: 'High' 

tools/mosesdecoder/biconcor/Alignment.cpp

@@ -0,0 +1,222 @@
+#include "Alignment.h"
+
+#include <fstream>
+#include <string>
+#include <cstdlib>
+#include <cstring>
+
+namespace
+{
+
+const int LINE_MAX_LENGTH = 10000;
+
+} // namespace
+
+using namespace std;
+
+void Alignment::Create(const string& fileName)
+{
+  ifstream textFile;
+  char line[LINE_MAX_LENGTH];
+
+  // count the number of words first;
+  textFile.open(fileName.c_str());
+
+  if (!textFile) {
+    cerr << "No such file or directory: " << fileName << endl;
+    exit(1);
+  }
+
+  istream *fileP = &textFile;
+  m_size = 0;
+  m_sentenceCount = 0;
+  while(!fileP->eof()) {
+    SAFE_GETLINE((*fileP), line, LINE_MAX_LENGTH, '\n');
+    if (fileP->eof()) break;
+    vector<string> alignmentSequence = Tokenize( line );
+    m_size += alignmentSequence.size();
+    m_sentenceCount++;
+  }
+  textFile.close();
+  cerr << m_size << " alignment points" << endl;
+
+  // allocate memory
+  m_array = (int*) calloc( sizeof(int), m_size*2 );
+  m_sentenceEnd = (INDEX*) calloc( sizeof( INDEX ), m_sentenceCount );
+
+  if (m_array == NULL) {
+    cerr << "Error: cannot allocate memory to m_array" << endl;
+    exit(1);
+  }
+
+  if (m_sentenceEnd == NULL) {
+    cerr << "Error: cannot allocate memory to m_sentenceEnd" << endl;
+    exit(1);
+  }
+
+  // fill the array
+  int alignmentPointIndex = 0;
+  int sentenceId = 0;
+
+  textFile.open(fileName.c_str());
+  if (!textFile) {
+    cerr << "Failed to open " << fileName << endl;
+    exit(1);
+  }
+
+  fileP = &textFile;
+  while(!fileP->eof()) {
+    SAFE_GETLINE((*fileP), line, LINE_MAX_LENGTH, '\n');
+    if (fileP->eof()) break;
+    vector<string> alignmentSequence = Tokenize( line );
+    for(size_t i=0; i<alignmentSequence.size(); i++) {
+      int s,t;
+      // cout << "scaning " << alignmentSequence[i].c_str() << endl;
+      if (! sscanf(alignmentSequence[i].c_str(), "%d-%d", &s, &t)) {
+        cerr << "WARNING: " << alignmentSequence[i] << " is a bad alignment point in sentence " << sentenceId << endl;
+      }
+      m_array[alignmentPointIndex++] = (char) s;
+      m_array[alignmentPointIndex++] = (char) t;
+    }
+    m_sentenceEnd[ sentenceId++ ] = alignmentPointIndex - 2;
+  }
+  textFile.close();
+  cerr << "done reading " << (alignmentPointIndex/2) << " alignment points, " << sentenceId << " sentences." << endl;
+}
+
+Alignment::Alignment()
+  : m_array(NULL),
+    m_sentenceEnd(NULL),
+    m_size(0),
+    m_sentenceCount(0) {}
+
+Alignment::~Alignment()
+{
+  if (m_array != NULL) {
+    free(m_array);
+  }
+  if (m_sentenceEnd != NULL) {
+    free(m_sentenceEnd);
+  }
+}
+
+vector<string> Alignment::Tokenize( const char input[] )
+{
+  vector< string > token;
+  bool betweenWords = true;
+  int start=0;
+  int i=0;
+  for(; input[i] != '\0'; i++) {
+    bool isSpace = (input[i] == ' ' || input[i] == '\t');
+
+    if (!isSpace && betweenWords) {
+      start = i;
+      betweenWords = false;
+    } else if (isSpace && !betweenWords) {
+      token.push_back( string( input+start, i-start ) );
+      betweenWords = true;
+    }
+  }
+  if (!betweenWords)
+    token.push_back( string( input+start, i-start ) );
+  return token;
+}
+
+bool Alignment::PhraseAlignment( INDEX sentence, int target_length,
+                                 int source_start, int source_end,
+                                 int &target_start, int &target_end,
+                                 int &pre_null, int &post_null )
+{
+  // get index for first alignment point
+  INDEX sentenceStart = 0;
+  if (sentence > 0) {
+    sentenceStart = m_sentenceEnd[ sentence-1 ] + 2;
+  }
+
+  // get target phrase boundaries
+  target_start = target_length;
+  target_end = 0;
+  for(INDEX ap = sentenceStart; ap <= m_sentenceEnd[ sentence ]; ap += 2 ) {
+    int source = m_array[ ap ];
+    if (source >= source_start && source <= source_end ) {
+      int target =  m_array[ ap+1 ];
+      if (target < target_start) target_start = target;
+      if (target > target_end )  target_end   = target;
+    }
+  }
+  if (target_start == target_length) {
+    return false; // done if no alignment points
+  }
+
+  // check consistency
+  for(INDEX ap = sentenceStart; ap <= m_sentenceEnd[ sentence ]; ap += 2 ) {
+    int target =  m_array[ ap+1 ];
+    if (target >= target_start && target <= target_end ) {
+      int source = m_array[ ap ];
+      if (source < source_start || source > source_end) {
+        return false; // alignment point out of range
+      }
+    }
+  }
+
+  // create array for unaligned words
+  for( int i=0; i<target_length; i++ ) {
+    m_unaligned[i] = true;
+  }
+  for(INDEX ap = sentenceStart; ap <= m_sentenceEnd[ sentence ]; ap += 2 ) {
+    int target =  m_array[ ap+1 ];
+    m_unaligned[ target ] = false;
+  }
+
+  // prior unaligned words
+  pre_null = 0;
+  for(int target = target_start-1; target >= 0 && m_unaligned[ target ]; target--) {
+    pre_null++;
+  }
+
+  // post unaligned words;
+  post_null = 0;
+  for(int target = target_end+1; target < target_length && m_unaligned[ target ]; target++) {
+    post_null++;
+  }
+  return true;
+}
+
+void Alignment::Save(const string& fileName ) const
+{
+  FILE *pFile = fopen ( (fileName + ".align").c_str() , "w" );
+  if (pFile == NULL) {
+    cerr << "Cannot open " << fileName << ".align" << endl;
+    exit(1);
+  }
+
+  fwrite( &m_size, sizeof(INDEX), 1, pFile );
+  fwrite( m_array, sizeof(int), m_size*2, pFile ); // corpus
+
+  fwrite( &m_sentenceCount, sizeof(INDEX), 1, pFile );
+  fwrite( m_sentenceEnd, sizeof(INDEX), m_sentenceCount, pFile); // sentence index
+  fclose( pFile );
+}
+
+void Alignment::Load(const string& fileName )
+{
+  FILE *pFile = fopen ( (fileName + ".align").c_str() , "r" );
+  if (pFile == NULL) {
+    cerr << "no such file or directory: " << fileName << ".align" << endl;
+    exit(1);
+  }
+
+  cerr << "loading from " << fileName << ".align" << endl;
+
+  fread( &m_size, sizeof(INDEX), 1, pFile );
+  cerr << "alignment points in corpus: " << m_size << endl;
+  m_array = (int*) calloc( sizeof(int), m_size*2 );
+  fread( m_array, sizeof(int), m_size*2, pFile ); // corpus
+
+  fread( &m_sentenceCount, sizeof(INDEX), 1, pFile );
+  cerr << "sentences in corpus: " << m_sentenceCount << endl;
+  m_sentenceEnd = (INDEX*) calloc( sizeof(INDEX), m_sentenceCount );
+  fread( m_sentenceEnd, sizeof(INDEX), m_sentenceCount, pFile); // sentence index
+  fclose( pFile );
+  cerr << "done loading\n";
+}

tools/mosesdecoder/biconcor/Alignment.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "Vocabulary.h"
+
+class Alignment
+{
+public:
+  typedef unsigned int INDEX;
+
+private:
+  int *m_array;
+  INDEX *m_sentenceEnd;
+  INDEX m_size;
+  INDEX m_sentenceCount;
+  char m_unaligned[ 256 ]; // here for speed (local to PhraseAlignment)
+
+  // No copying allowed.
+  Alignment(const Alignment&);
+  void operator=(const Alignment&);
+
+public:
+  Alignment();
+  ~Alignment();
+
+  void Create(const std::string& fileName );
+  bool PhraseAlignment( INDEX sentence, int target_length,
+                        int source_start, int source_end,
+                        int &target_start, int &target_end,
+                        int &pre_null, int &post_null );
+  void Load(const std::string& fileName );
+  void Save(const std::string& fileName ) const;
+  std::vector<std::string> Tokenize( const char input[] );
+
+  INDEX GetSentenceStart( INDEX sentence ) const {
+    if (sentence == 0) return 0;
+    return m_sentenceEnd[ sentence-1 ] + 2;
+  }
+  INDEX GetNumberOfAlignmentPoints( INDEX sentence ) const {
+    return ( m_sentenceEnd[ sentence ] - GetSentenceStart( sentence ) ) / 2;
+  }
+  int GetSourceWord( INDEX sentence, INDEX alignment_point ) const {
+    return m_array[ GetSentenceStart( sentence ) + alignment_point*2 ];
+  }
+  int GetTargetWord( INDEX sentence, INDEX alignment_point ) const {
+    return m_array[ GetSentenceStart( sentence ) + alignment_point*2 + 1 ];
+  }
+};

tools/mosesdecoder/biconcor/CMakeLists.txt

@@ -0,0 +1,5 @@
+project(biconcor)
+
+FILE(GLOB biconcor_source *.cpp)
+
+add_executable(biconcor ${biconcor_source})

tools/mosesdecoder/biconcor/Jamfile

@@ -0,0 +1,2 @@
+exe biconcor : Vocabulary.cpp SuffixArray.cpp TargetCorpus.cpp Alignment.cpp Mismatch.cpp PhrasePair.cpp PhrasePairCollection.cpp biconcor.cpp base64.cpp ;
+exe phrase-lookup : Vocabulary.cpp SuffixArray.cpp phrase-lookup.cpp ;

tools/mosesdecoder/biconcor/Mismatch.cpp

@@ -0,0 +1,292 @@
+#include "Mismatch.h"
+
+#include <fstream>
+#include <iostream>
+#include <cstring>
+#include <string>
+#include <cstdlib>
+
+#include "SuffixArray.h"
+#include "TargetCorpus.h"
+#include "Alignment.h"
+#include "Vocabulary.h"
+
+using namespace std;
+
+enum {
+  UNANNOTATED = 0,
+  PRE_ALIGNED = 1,
+  POST_ALIGNED = 2,
+  UNALIGNED = 3,
+  MISALIGNED = 4,
+  ALIGNED = 5
+};
+
+Mismatch::Mismatch( SuffixArray *sa, TargetCorpus *tc, Alignment *a, INDEX sentence_id, INDEX position, int source_length, int target_length, int source_start, int source_end )
+  :m_suffixArray(sa)
+  ,m_targetCorpus(tc)
+  ,m_alignment(a)
+  ,m_sentence_id(sentence_id)
+  ,m_source_length(source_length)
+  ,m_target_length(target_length)
+  ,m_source_position(position)
+  ,m_source_start(source_start)
+  ,m_source_end(source_end)
+  ,m_unaligned(true)
+{
+  // initialize unaligned indexes
+  for (int i = 0; i < m_source_length; i++) {
+    m_source_unaligned[i] = true;
+  }
+  for (int i = 0; i < m_target_length; i++) {
+    m_target_unaligned[i] = true;
+  }
+  m_num_alignment_points =
+    m_alignment->GetNumberOfAlignmentPoints( sentence_id );
+  for(INDEX ap=0; ap<m_num_alignment_points; ap++) {
+    m_source_unaligned[ (int)m_alignment->GetSourceWord( sentence_id, ap ) ] = false;
+    m_target_unaligned[ (int)m_alignment->GetTargetWord( sentence_id, ap ) ] = false;
+  }
+  for(int i = source_start; i <= source_end; i++) {
+    if (!m_source_unaligned[ i ]) {
+      m_unaligned = false;
+    }
+  }
+}
+
+Mismatch::~Mismatch () {}
+
+void Mismatch::PrintClippedHTML( ostream* out, int width )
+{
+  int source_annotation[256], target_annotation[256];
+  vector< string > label_class;
+  label_class.push_back( "" );
+  label_class.push_back( "mismatch_pre_aligned" );
+  label_class.push_back( "mismatch_post_aligned" );
+  label_class.push_back( "null_aligned" );
+  label_class.push_back( "mismatch_misaligned" );
+  label_class.push_back( "mismatch_aligned" );
+
+  for(int i=0; i<m_source_length; i++) source_annotation[i] = UNANNOTATED;
+  for(int i=0; i<m_target_length; i++) target_annotation[i] = UNANNOTATED;
+
+  if (m_unaligned) {
+    // find alignment points for prior and next word(s) and
+    // center target phrase around those.
+    bool found_aligned = false;
+    for(int i=1; i<m_source_length && !found_aligned; i++) {
+      if (m_source_start-i >= 0) {
+        int word_id =  m_source_start-i;
+        source_annotation[ word_id ] = UNALIGNED;
+        if (!m_source_unaligned[ word_id ]) {
+          found_aligned = true;
+          LabelSourceMatches( source_annotation, target_annotation, word_id, PRE_ALIGNED );
+        }
+      }
+
+      if (m_source_end+i < m_source_length) {
+        int word_id = m_source_end+i;
+        source_annotation[ word_id ] = UNALIGNED;
+        if (!m_source_unaligned[ word_id ]) {
+          found_aligned = true;
+          LabelSourceMatches( source_annotation, target_annotation, word_id, POST_ALIGNED );
+        }
+      }
+    }
+
+  }
+  // misalignment
+  else {
+    // label aligned output words
+    for(int i=m_source_start; i<=m_source_end; i++)
+      LabelSourceMatches( source_annotation, target_annotation, i, ALIGNED );
+
+    // find first and last
+    int target_start = -1;
+    int target_end = -1;
+    for(int i=0; i<m_target_length; i++)
+      if (target_annotation[i] == ALIGNED) {
+        if (target_start == -1)
+          target_start = i;
+        target_end = i;
+      }
+    // go over all enclosed target words
+    for(int i=target_start; i<=target_end; i++) {
+      // label other target words as unaligned or misaligned
+      if (m_target_unaligned[ i ])
+        target_annotation[ i ] = UNALIGNED;
+      else {
+        if (target_annotation[ i ] != ALIGNED)
+          target_annotation[ i ] = MISALIGNED;
+        // loop over aligned source words
+        for(INDEX ap=0; ap<m_num_alignment_points; ap++) {
+          if (m_alignment->GetTargetWord( m_sentence_id, ap ) == i) {
+            int source_word = m_alignment->GetSourceWord( m_sentence_id, ap );
+            // if not part of the source phrase -> also misaligned
+            if (source_word < m_source_start || source_word > m_source_end)
+              source_annotation[ source_word ] = MISALIGNED;
+          }
+        }
+      }
+    }
+    // closure
+    bool change = true;
+    while(change) {
+      change = false;
+      for(INDEX ap=0; ap<m_num_alignment_points; ap++) {
+        int source_word = m_alignment->GetSourceWord( m_sentence_id, ap );
+        int target_word = m_alignment->GetTargetWord( m_sentence_id, ap );
+        if (source_annotation[source_word] != UNANNOTATED &&
+            target_annotation[target_word] == UNANNOTATED) {
+          target_annotation[target_word] = MISALIGNED;
+          change = true;
+        }
+        if (source_annotation[source_word] == UNANNOTATED &&
+            target_annotation[target_word] != UNANNOTATED) {
+          source_annotation[source_word] = MISALIGNED;
+          change = true;
+        }
+      }
+    }
+  }
+
+  // print source
+  // shorten source context if too long
+  int sentence_start = m_source_position - m_source_start;
+  int context_space = width/2;
+  for(int i=m_source_start; i<=m_source_end; i++)
+    context_space -= m_suffixArray->GetWord( sentence_start + i ).size() + 1;
+  context_space /= 2;
+
+  int remaining = context_space;
+  int start_word = m_source_start;
+  for(; start_word>0 && remaining>0; start_word--)
+    remaining -= m_suffixArray->GetWord( sentence_start + start_word-1 ).size() + 1;
+  if (remaining<0 || start_word == -1) start_word++;
+
+  remaining = context_space;
+  int end_word = m_source_end;
+  for(; end_word<m_source_length && remaining>0; end_word++)
+    remaining -= m_suffixArray->GetWord( sentence_start + end_word ).size() + 1;
+  end_word--;
+
+  // output with markup
+  *out << "<tr><td class=\"pp_source_left\">";
+  char current_label = UNANNOTATED;
+  if (start_word>0) {
+    current_label = source_annotation[start_word-1];
+    *out << "... ";
+  }
+  for(int i=start_word; i<=end_word; i++) {
+    // change to phrase block
+    if (i == m_source_start) {
+      if (current_label != UNANNOTATED && i!=start_word)
+        *out << "</span>";
+      *out << "</td><td class=\"pp_source\">";
+      current_label = UNANNOTATED;
+    }
+
+    // change to labeled word
+    else if (source_annotation[i] != current_label &&
+             source_annotation[i] != ALIGNED) {
+      if (current_label != UNANNOTATED && i!=start_word)
+        *out << "</span>";
+      if (source_annotation[i] != UNANNOTATED)
+        *out << "<span class=\""
+             << label_class[ source_annotation[i] ]
+             << "\">";
+      current_label = source_annotation[i];
+    }
+
+    // output word
+    *out << m_suffixArray->GetWord( sentence_start + i ) << " ";
+
+    // change to right context block
+    if (i == m_source_end) {
+      *out << "</td><td class=\"pp_source_right\">";
+      current_label = UNANNOTATED;
+    }
+  }
+
+  if (current_label != UNANNOTATED && end_word>m_source_end)
+    *out << "</span>";
+  if (end_word<m_source_length-1)
+    *out << "... ";
+
+  // print target
+  // shorten target context if too long
+  int target_start = -1;
+  int target_end=0;
+  for(int i=0; i<m_target_length; i++)
+    if (target_annotation[i] != UNANNOTATED) {
+      if (target_start == -1)
+        target_start = i;
+      target_end = i;
+    }
+
+  context_space = width/2;
+  for(int i=target_start; i<=target_end; i++)
+    context_space -= m_targetCorpus->GetWord( m_sentence_id, i ).size() + 1;
+  while (context_space < 0) { // shorten matched part, if too long
+    context_space +=
+      m_targetCorpus->GetWord( m_sentence_id, target_start ).size() +
+      m_targetCorpus->GetWord( m_sentence_id, target_end ).size() + 2;
+    target_start++;
+    target_end--;
+  }
+  context_space /= 2;
+
+  remaining = context_space;
+  start_word = target_start;
+  for(; start_word>0 && remaining>0; start_word--) {
+    //cerr << "remaining: " << remaining << ", start_word: " << start_word << endl;
+    remaining -= m_targetCorpus->GetWord( m_sentence_id, start_word-1 ).size() + 1;
+  }
+  if (remaining<0 || start_word == -1) start_word++;
+
+  remaining = context_space;
+  end_word = target_end;
+  for(; end_word<m_target_length && remaining>0; end_word++) {
+    //cerr << "remaining: " << remaining << ", end_word: " << end_word << endl;
+    remaining -= m_targetCorpus->GetWord( m_sentence_id, end_word ).size() + 1;
+  }
+  end_word--;
+
+  // output with markup
+  *out << "</td><td class=\"mismatch_target\">";
+  current_label = UNANNOTATED;
+  if (start_word>0) {
+    current_label = target_annotation[start_word-1];
+    *out << "... ";
+  }
+  for(int i=start_word; i<=end_word; i++) {
+    if (target_annotation[i] != current_label) {
+      if (current_label != UNANNOTATED && i!=start_word)
+        *out << "</span>";
+      if (target_annotation[i] != UNANNOTATED)
+        *out << "<span class=\""
+             << label_class[ target_annotation[i] ]
+             << "\">";
+      current_label = target_annotation[i];
+    }
+
+    // output word
+    *out << m_targetCorpus->GetWord( m_sentence_id, i ) << " ";
+  }
+
+  if (current_label != UNANNOTATED && end_word>target_end)
+    *out << "</span>";
+  if (end_word<m_target_length-1)
+    *out << "... ";
+  *out << "</td></tr>";
+}
+
+void Mismatch::LabelSourceMatches(int *source_annotation, int *target_annotation, int source_id, int label )
+{
+  for(INDEX ap=0; ap<m_num_alignment_points; ap++) {
+    if (m_alignment->GetSourceWord( m_sentence_id, ap ) == source_id) {
+      source_annotation[ source_id ] = label;
+      target_annotation[ m_alignment->GetTargetWord( m_sentence_id, ap ) ] = label;
+    }
+  }
+}

tools/mosesdecoder/biconcor/Mismatch.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <iosfwd>
+
+class Alignment;
+class SuffixArray;
+class TargetCorpus;
+
+class Mismatch
+{
+public:
+  typedef unsigned int INDEX;
+
+private:
+  SuffixArray *m_suffixArray;
+  TargetCorpus *m_targetCorpus;
+  Alignment *m_alignment;
+  INDEX m_sentence_id;
+  INDEX m_num_alignment_points;
+  int m_source_length;
+  int m_target_length;
+  INDEX m_source_position;
+  int m_source_start;
+  int m_source_end;
+  bool m_source_unaligned[ 256 ];
+  bool m_target_unaligned[ 256 ];
+  bool m_unaligned;
+
+  // No copying allowed.
+  Mismatch(const Mismatch&);
+  void operator=(const Mismatch&);
+
+public:
+  Mismatch( SuffixArray *sa, TargetCorpus *tc, Alignment *a, INDEX sentence_id, INDEX position, int source_length, int target_length, int source_start, int source_end );
+  ~Mismatch();
+
+  bool Unaligned() const {
+    return m_unaligned;
+  }
+  void PrintClippedHTML(std::ostream* out, int width );
+  void LabelSourceMatches(int *source_annotation, int *target_annotation, int source_id, int label );
+};

tools/mosesdecoder/biconcor/PhrasePair.cpp

@@ -0,0 +1,300 @@
+#include "PhrasePair.h"
+
+#include <iostream>
+#include "TargetCorpus.h"
+#include "Alignment.h"
+#include "Vocabulary.h"
+#include "SuffixArray.h"
+
+using namespace std;
+
+void PhrasePair::Print( ostream* out ) const
+{
+  // source
+  int sentence_start = m_source_position - m_source_start;
+  char source_length = m_suffixArray->GetSentenceLength( m_suffixArray->GetSentence( m_source_position ) );
+
+  for( char i=0; i<source_length; i++ ) {
+    if (i>0) *out << " ";
+    *out << m_suffixArray->GetWord( sentence_start + i );
+  }
+
+  // target
+  *out << " |||";
+  for( char i=0; i<m_target_length; i++ ) {
+    *out << " " << m_targetCorpus->GetWord( m_sentence_id, i);
+  }
+
+  // source span
+  *out << " ||| " << (int)m_source_start << " " << (int)m_source_end;
+
+  // target span
+  *out << " ||| " << (int)m_target_start << " " << (int)m_target_end;
+
+  // word alignment
+  *out << " |||";
+
+  INDEX ap_points = m_alignment->GetNumberOfAlignmentPoints( m_sentence_id );
+  for( INDEX i=0; i<ap_points; i++) {
+    *out << " " << m_alignment->GetSourceWord( m_sentence_id, i )
+         << "-" << m_alignment->GetTargetWord( m_sentence_id, i );
+  }
+
+  *out << endl;
+}
+
+void PhrasePair::PrintPretty( ostream* out, int width ) const
+{
+  vector< WORD_ID >::const_iterator t;
+
+  // source
+  int sentence_start = m_source_position - m_source_start;
+  size_t source_width = (width-3)/2;
+  string source_pre = "";
+  string source = "";
+  string source_post = "";
+  for( size_t space=0; space<source_width/2; space++ ) source_pre += " ";
+  for( char i=0; i<m_source_start; i++ ) {
+    source_pre += " " + m_suffixArray->GetWord( sentence_start + i );
+  }
+  for( char i=m_source_start; i<=m_source_end; i++ ) {
+    if (i>m_source_start) source += " ";
+    source += m_suffixArray->GetWord( sentence_start + i );
+  }
+  char source_length = m_suffixArray->GetSentenceLength( m_suffixArray->GetSentence( m_source_position ) );
+  for( char i=m_source_end+1; i<source_length; i++ ) {
+    if (i>m_source_end+1) source_post += " ";
+    source_post += m_suffixArray->GetWord( sentence_start + i );
+  }
+  for( size_t space=0; space<source_width/2; space++ ) source_post += " ";
+
+  size_t source_pre_width = (source_width-source.size()-2)/2;
+  size_t source_post_width = (source_width-source.size()-2+1)/2;
+
+  if (source.size() > (size_t)width) {
+    source_pre_width = 0;
+    source_post_width = 0;
+  }
+
+  *out << source_pre.substr( source_pre.size()-source_pre_width, source_pre_width ) << " "
+       << source.substr( 0, source_width -2 ) << " "
+       << source_post.substr( 0, source_post_width ) << " | ";
+
+  // target
+  size_t target_width = (width-3)/2;
+  string target_pre = "";
+  string target = "";
+  string target_post = "";
+  for( size_t space=0; space<target_width/2; space++ ) target_pre += " ";
+  for( char i=0; i<m_target_start; i++ ) {
+    target_pre += " " + m_targetCorpus->GetWord( m_sentence_id, i);
+  }
+  for( char i=m_target_start; i<=m_target_end; i++ ) {
+    if (i>m_target_start) target += " ";
+    target += m_targetCorpus->GetWord( m_sentence_id, i);
+  }
+  for( char i=m_target_end+1; i<m_target_length; i++ ) {
+    if (i>m_target_end+1) target_post += " ";
+    target_post += m_targetCorpus->GetWord( m_sentence_id, i);
+  }
+
+  size_t target_pre_width = (target_width-target.size()-2)/2;
+  size_t target_post_width = (target_width-target.size()-2+1)/2;
+
+  if (target.size() > (size_t)width) {
+    target_pre_width = 0;
+    target_post_width = 0;
+  }
+
+  *out << target_pre.substr( target_pre.size()-target_pre_width, target_pre_width ) << " "
+       << target.substr( 0, target_width -2 ) << " "
+       << target_post.substr( 0, target_post_width ) << endl;
+}
+
+void PhrasePair::PrintTarget( ostream* out ) const
+{
+  for( char i=m_target_start; i<=m_target_end; i++ ) {
+    if (i>m_target_start) *out << " ";
+    *out << m_targetCorpus->GetWord( m_sentence_id, i);
+  }
+}
+
+void PhrasePair::PrintHTML( ostream* out ) const
+{
+  // source
+  int sentence_start = m_source_position - m_source_start;
+  char source_length = m_suffixArray->GetSentenceLength( m_suffixArray->GetSentence( m_source_position ) );
+
+  *out << "<tr><td align=right class=\"pp_source_left\">";
+  for( char i=0; i<m_source_start; i++ ) {
+    if (i>0) *out << " ";
+    *out << m_suffixArray->GetWord( sentence_start + i );
+  }
+  *out << "</td><td class=\"pp_source\">";
+  for( char i=m_source_start; i<=m_source_end; i++ ) {
+    if (i>m_source_start) *out << " ";
+    *out << m_suffixArray->GetWord( sentence_start + i );
+  }
+  *out << "</td><td class=\"pp_source_right\">";
+  for( char i=m_source_end+1; i<source_length; i++ ) {
+    if (i>m_source_end+1) *out << " ";
+    *out << m_suffixArray->GetWord( sentence_start + i );
+  }
+
+  // target
+  *out << "</td><td class=\"pp_target_left\">";
+  for( char i=0; i<m_target_start; i++ ) {
+    if (i>0) *out << " ";
+    *out << m_targetCorpus->GetWord( m_sentence_id, i);
+  }
+  *out << "</td><td class=\"pp_target\">";
+  for( char i=m_target_start; i<=m_target_end; i++ ) {
+    if (i>m_target_start) *out << " ";
+    *out << m_targetCorpus->GetWord( m_sentence_id, i);
+  }
+  *out << "</td><td class=\"pp_target_right\">";
+  for( char i=m_target_end+1; i<m_target_length; i++ ) {
+    if (i>m_target_end+1) *out << " ";
+    *out << m_targetCorpus->GetWord( m_sentence_id, i);
+  }
+  *out << "</td></tr>\n";
+}
+
+void PhrasePair::PrintClippedHTML( ostream* out, int width ) const
+{
+  vector< WORD_ID >::const_iterator t;
+
+  // source
+  int sentence_start = m_source_position - m_source_start;
+  size_t source_width = (width+1)/2;
+  string source_pre = "";
+  string source = "";
+  string source_post = "";
+  for( char i=0; i<m_source_start; i++ ) {
+    source_pre += " " + m_suffixArray->GetWord( sentence_start + i );
+  }
+  for( char i=m_source_start; i<=m_source_end; i++ ) {
+    if (i>m_source_start) source += " ";
+    source += m_suffixArray->GetWord( sentence_start + i );
+  }
+  char source_length = m_suffixArray->GetSentenceLength( m_suffixArray->GetSentence( m_source_position ) );
+  for( char i=m_source_end+1; i<source_length; i++ ) {
+    if (i>m_source_end+1) source_post += " ";
+    source_post += m_suffixArray->GetWord( sentence_start + i );
+  }
+  size_t source_pre_width = (source_width-source.size())/2;
+  size_t source_post_width = (source_width-source.size()+1)/2;
+
+  // if phrase is too long, don't show any context
+  if (source.size() > (size_t)width) {
+    source_pre_width = 0;
+    source_post_width = 0;
+  }
+  // too long -> truncate and add "..."
+  if (source_pre.size() > source_pre_width) {
+    // first skip up to a space
+    while(source_pre_width>0 &&
+          source_pre.substr(source_pre.size()-source_pre_width,1) != " ") {
+      source_pre_width--;
+    }
+    source_pre = "..." + source_pre.substr( source_pre.size()-source_pre_width, source_pre_width );
+  }
+  if (source_post.size() > source_post_width) {
+    while(source_post_width>0 &&
+          source_post.substr(source_post_width-1,1) != " ") {
+      source_post_width--;
+    }
+    source_post = source_post.substr( 0, source_post_width ) + "...";
+  }
+
+  *out << "<tr><td class=\"pp_source_left\">"
+       << source_pre
+       << "</td><td class=\"pp_source\">"
+       << source.substr( 0, source_width -2 )
+       << "</td><td class=\"pp_source_right\">"
+       << source_post
+       << "</td>";
+
+  // target
+  size_t target_width = width/2;
+  string target_pre = "";
+  string target = "";
+  string target_post = "";
+  size_t target_pre_null_width = 0;
+  size_t target_post_null_width = 0;
+  for( char i=0; i<m_target_start; i++ ) {
+    WORD word = m_targetCorpus->GetWord( m_sentence_id, i);
+    target_pre += " " + word;
+    if (i >= m_target_start-m_pre_null)
+      target_pre_null_width += word.size() + 1;
+  }
+  for( char i=m_target_start; i<=m_target_end; i++ ) {
+    if (i>m_target_start) target += " ";
+    target += m_targetCorpus->GetWord( m_sentence_id, i);
+  }
+  for( char i=m_target_end+1; i<m_target_length; i++ ) {
+    if (i>m_target_end+1) target_post += " ";
+    WORD word = m_targetCorpus->GetWord( m_sentence_id, i);
+    target_post += word;
+    if (i-(m_target_end+1) < m_post_null) {
+      target_post_null_width += word.size() + 1;
+    }
+  }
+
+  size_t target_pre_width = (target_width-target.size())/2;
+  size_t target_post_width = (target_width-target.size()+1)/2;
+
+  if (target.size() > (size_t)width) {
+    target_pre_width = 0;
+    target_post_width = 0;
+  }
+
+  if (target_pre.size() < target_pre_width)
+    target_pre_width = target_pre.size();
+  else {
+    while(target_pre_width>0 &&
+          target_pre.substr(target_pre.size()-target_pre_width,1) != " ") {
+      target_pre_width--;
+    }
+    target_pre = "..." + target_pre.substr( target_pre.size()-target_pre_width, target_pre_width );
+  }
+
+  if (target_post.size() < target_post_width) {
+    target_post_width = target_post.size();
+  } else {
+    while(target_post_width>0 &&
+          target_post.substr(target_post_width-1,1) != " ") {
+      target_post_width--;
+    }
+    target_post = target_post.substr( 0, target_post_width ) + "...";
+  }
+
+  if (m_pre_null) {
+    //cerr << endl << "target_pre_width=" << target_pre_width << ", target_pre_null_width=" << target_pre_null_width << ", target_pre.size()=" << target_pre.size() << endl;
+    if (target_pre_width < target_pre.size())
+      target_pre_null_width -= target_pre.size()-target_pre_width;
+    target_pre = target_pre.substr(0,target_pre_width-target_pre_null_width)
+                 + "<span class=\"null_aligned\">"
+                 + target_pre.substr(target_pre_width-target_pre_null_width)
+                 + "</span>";
+  }
+  if (m_post_null) {
+    //cerr << endl << "target_post_width=" << target_post_width << ", target_post_null_width=" << target_post_null_width << ", target_post.size()=" << target_post.size() << endl;
+    if (target_post_null_width > target_post.size()) {
+      target_post_null_width = target_post.size();
+    }
+    target_post = "<span class=\"null_aligned\">"
+                  + target_post.substr(0,target_post_null_width)
+                  + "</span>"
+                  + target_post.substr(target_post_null_width);
+  }
+
+  *out << "<td class=\"pp_target_left\">"
+       << target_pre
+       << "</td><td class=\"pp_target\">"
+       << target.substr( 0, target_width -2 )
+       << "</td><td class=\"pp_target_right\">"
+       << target_post
+       << "</td></tr>"<< endl;
+}
+