/*
 * Created on May 12, 2005
 *
 */
package com.kellegous.mas864.clusterss.nlp;

import java.util.HashSet;
import java.util.Set;

/**
 * @author knorton
 * Copyright (C) 2005 MIT Media Lab, All Right Reserved.
 */
public class WordExtractor {
    private static HashSet	stopDic = null;
    private static Stemmer	stemmer = null;
    
    private static int		LENGTH_MIN=3;
    private static int		LENGTH_MAX=Integer.MAX_VALUE;
    
    public static boolean	DO_LENGTH_CHECKING = true;
    public static boolean	DO_STOP_CHECKING = true;
    public static boolean DO_STEMMING = true;

    static {
        stemmer = new Stemmer();
        loadStopList();
    }
    
    public static void loadStopList() {
        HashSet	s = new HashSet();
        for (int i=0;i<StopList.WORDS.length;i++)
            s.add(StopList.WORDS[i]);
        stopDic = s;
    }
    
    public static String[] extract(String data) {
        HashSet 	s = extractAsSet(data);
        String[] a = new String[s.size()];
        s.toArray(a);
        return a;
    }
    
    private static void validateAndAddWordToSet(String wd, Set set) {
        if (wd == null)
            return;
        wd = wd.toLowerCase();

        if (DO_LENGTH_CHECKING && (wd.length()<LENGTH_MIN || wd.length()>LENGTH_MAX))
            return;
        
        String sd = wd;
        if (DO_STEMMING) {
            stemmer.add(wd.toCharArray(),wd.length());
            stemmer.stem();
            sd = stemmer.toString();
        }
        
        if (set.contains(sd))
            return;
        
        if (DO_STOP_CHECKING && stopDic != null && stopDic.contains(wd))
            return;
        
        set.add(sd);
    }
    
    public static HashSet extractAsSet(String data) {
        HashSet 		wl = new HashSet();
        StringBuffer wd = new StringBuffer();
        int n = data.length();
        for (int i=0;i<n;i++) {
            char c = data.charAt(i);
            if (Character.isLetter(c)) {
                wd.append(c);
            }
            else {
                validateAndAddWordToSet(wd.toString(),wl);
                wd = new StringBuffer();
            }
        }
        if (wd.length()>0)
            validateAndAddWordToSet(wd.toString(),wl);
        
        return wl;
    }
}
