Projects >> joshua >>49ff6088e20376fa77cf2f9dcf90002191809e00

Chunk
Conflicting content 
   */
  }

  /**
<<<<<<< HEAD
=======
   * Retained to maintain backward compatibility Uses the lm lines in the Joshua config file which
   * are not defined as feature functions to create new LMs
   * 
   * @param args
   * @throws IOException
   */
  private void initializeLanguageModels() throws IOException {

    if (joshuaConfiguration.lms.size() > 0) {
      Decoder.LOG(1, "You seem to be using an old version of the Joshua config file");
      Decoder.LOG(1, "Language models should be defined as regular feature functions.");

      // Only initialize if necessary
      if (this.languageModels == null) {
        this.languageModels = new ArrayList();
      }
      // lm = kenlm 5 0 0 100 file
      for (String lmLine : joshuaConfiguration.lms) {
        String[] tokens = lmLine.trim().split("\\s+");

        HashMap argMap = new HashMap();
        argMap.put("lm_type", tokens[0]);
        argMap.put("lm_order", tokens[1]);
        argMap.put("minimizing", tokens[2]);
        argMap.put("lm_file", tokens[5]);
        initializeLanguageModel(argMap);
      }
    }
  }

  /**
   * Initializes a language model and adds it as a feature
   * 
   * @param argMap A map of arguments supplied top the lm feature function throught the Joshua
   *          config file
   * @throws IOException 
  private void initializeLanguageModel(HashMap argMap) throws IOException {
    if (this.languageModels == null) {
      this.languageModels = new ArrayList();
    }
    String lm_type = argMap.get("lm_type");
    int lm_order = Integer.parseInt(argMap.get("lm_order"));
    boolean minimizing = Boolean.parseBoolean(argMap.get("minimizing"));
    String lm_file = argMap.get("lm_file");

    if (lm_type.equals("kenlm")) {
      KenLM lm = new KenLM(lm_order, lm_file, minimizing);
      this.languageModels.add(lm);
      Vocabulary.registerLanguageModel(lm);
      Vocabulary.id(joshuaConfiguration.default_non_terminal);
      
      if (argMap.containsKey("lm_class") && argMap.containsKey("class_map")) {
        // This is a Class LM 
        addLMFeature(lm, true, argMap.get("class_map"));
      }
      else {
        addLMFeature(lm, false, null);
      }

    } else if (lm_type.equals("berkeleylm")) {
      LMGrammarBerkeley lm = new LMGrammarBerkeley(lm_order, lm_file);
      this.languageModels.add(lm);
      Vocabulary.registerLanguageModel(lm);
      Vocabulary.id(joshuaConfiguration.default_non_terminal);
      addLMFeature(lm, false, null);

    } else if (lm_type.equals("none")) {
      ; // do nothing

    } else {
      Decoder.LOG(1, "WARNING: using built-in language model; you probably didn't intend this");
      Decoder.LOG(1, "Valid lm types are 'kenlm', 'berkeleylm', 'none'");
    }
  }

  private void addLMFeature(NGramLanguageModel lm, boolean isClass, String classFile)
      throws IOException {
    if (lm instanceof KenLM && lm.isMinimizing()) {
      KenLMFF newFeatureFunction = new KenLMFF(weights, (KenLM) lm, joshuaConfiguration, isClass);
      newFeatureFunction.setClassMap(classFile);
      this.featureFunctions.add(newFeatureFunction);
      
    } else {
      this.featureFunctions.add(new LanguageModelFF(weights, lm, joshuaConfiguration, false));
    }
  }

  /**
>>>>>>> 3e38f504a22b9ef9a9f68301b6daf2cbd6da07db
   * Initializes translation grammars Retained for backward compatibility
   * 
   * @param ownersSeen Records which PhraseModelFF's have been instantiated (one is needed for each
Solution content 
  }

  /**
   * Initializes translation grammars Retained for backward compatibility
   * 
   * @param ownersSeen Records which PhraseModelFF's have been instantiated (one is needed for each
File
Decoder.java
Developer's decision
Version 1
Kind of conflict
Comment
Method declaration
Chunk
Conflicting content 
    }
   * We cache the weight of the feature since there is only one.
   */
  protected float weight;
<<<<<<< HEAD
  protected String type;
  protected String path;
=======

  /* The config */
  protected JoshuaConfiguration config;
  
  /**
   * 
   */
  private boolean isClassLM;
  private ClassMap classMap;
  
  protected class ClassMap {

    private HashMap classMap;

    public ClassMap(String file_name) throws IOException {
      this.classMap = new HashMap();
      read(file_name);
    }

    public int getClassID(int wordID) {
      if (this.classMap.containsKey(wordID)) {
        return this.classMap.get(wordID);
      } else {
        // OOV wrt the class map
        return -1;
      }

    /**
     * Reads a class map from file.
     * 
     * @param file_name
     * @throws IOException
     */
    private void read(String file_name) throws IOException {

      File class_file = new File(file_name);
      BufferedReader br = new BufferedReader(new FileReader(class_file));
      String line;

      while ((line = br.readLine()) != null) {
        String[] lineComp = line.trim().split("\\s+");
        this.classMap.put(Vocabulary.id(lineComp[0]), Integer.parseInt(lineComp[1]));
      }
      br.close();
    }

  }
>>>>>>> 3e38f504a22b9ef9a9f68301b6daf2cbd6da07db

  /**
   *
Solution content 
   * We cache the weight of the feature since there is only one.
   */
  protected float weight;
  protected String type;
  protected String path;

  /* Whether this is a class-based LM */
  private boolean isClassLM;
  private ClassMap classMap;
  
  protected class ClassMap {

    private final int OOV_id = 10;
    private HashMap classMap;

    public ClassMap(String file_name) throws IOException {
      this.classMap = new HashMap();
      read(file_name);
    }

    public int getClassID(int wordID) {
      if (this.classMap.containsKey(wordID)) {
        return this.classMap.get(wordID);
      } else {
        return OOV_id;
      }
    }

    /**
     * Reads a class map from file.
     * 
     * @param file_name
     * @throws IOException
     */
    private void read(String file_name) throws IOException {

      File class_file = new File(file_name);
      BufferedReader br = new BufferedReader(new FileReader(class_file));
      String line;

      while ((line = br.readLine()) != null) {
        String[] lineComp = line.trim().split("\\s+");
        this.classMap.put(Vocabulary.id(lineComp[0]), Integer.parseInt(lineComp[1]));
      }
      br.close();
    }

  }
File
LanguageModelFF.java
Developer's decision
Manual
Kind of conflict
Attribute
Class declaration
Comment
Chunk
Conflicting content 
  /**
   *
   */
<<<<<<< HEAD
  public LanguageModelFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
    super(weights, String.format("lm_%d", LanguageModelFF.LM_INDEX++), args, config);
=======
  public LanguageModelFF(FeatureVector weights, NGramLanguageModel lm, JoshuaConfiguration config, boolean isClassLM) {
    super(weights, String.format("lm_%d", LanguageModelFF.LM_INDEX++));
    this.languageModel = lm;
    this.ngramOrder = lm.getOrder();
    this.config = config;
    this.isClassLM = isClassLM;
>>>>>>> 3e38f504a22b9ef9a9f68301b6daf2cbd6da07db
    
    this.type = parsedArgs.get("lm_type");
    this.ngramOrder = Integer.parseInt(parsedArgs.get("lm_order"));
Solution content 
  public LanguageModelFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
    super(weights, String.format("lm_%d", LanguageModelFF.LM_INDEX++), args, config);

    this.type = parsedArgs.get("lm_type");
    this.ngramOrder = Integer.parseInt(parsedArgs.get("lm_order"));
File
LanguageModelFF.java
Developer's decision
Version 1
Kind of conflict
Attribute
Method invocation
Method signature
Variable
Chunk
Conflicting content 
  // maps from sentence numbers to KenLM-side pools used to allocate state
  private static final ConcurrentHashMap poolMap = new ConcurrentHashMap();

<<<<<<< HEAD:src/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
  public StateMinimizingLanguageModel(FeatureVector weights, String[] args, JoshuaConfiguration config) {
    super(weights, args, config);
    this.name = "StateMinimizingLanguageModel";

    this.type = "kenlm";
=======
  public KenLMFF(FeatureVector weights, KenLM lm, JoshuaConfiguration config, boolean isClass) {
    super(weights, lm, config, isClass);
>>>>>>> 3e38f504a22b9ef9a9f68301b6daf2cbd6da07db:src/joshua/decoder/ff/lm/KenLMFF.java
  }

  /**
Solution content 
  // maps from sentence numbers to KenLM-side pools used to allocate state
  private static final ConcurrentHashMap poolMap = new ConcurrentHashMap();

  public StateMinimizingLanguageModel(FeatureVector weights, String[] args, JoshuaConfiguration config) {
    super(weights, args, config);
    this.name = "StateMinimizingLanguageModel";

    this.type = "kenlm";
  }

  /**
File
StateMinimizingLanguageModel.java
Developer's decision
Version 1
Kind of conflict
Method invocation
Method signature