package src.java.org.apache.lucene.analysis.snowball;
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
import java.io.*;
import junit.framework.*;
import junit.framework.TestCase;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Hashtable;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.Token;
public class SnowballAnalyzerTest extends TestCase {
public SnowballAnalyzerTest(String name) {
super(name);
}
public void assertAnalyzesTo(Analyzer a, String input, String[] output) throws Exception {
TokenStream ts = a.tokenStream("dummy", new StringReader(input));
for (int i=0; i<output.length; i++) {
Token t = ts.next();
assertNotNull(t);
assertEquals(output[i], t.termText());
}
assertNull(ts.next());
ts.close();
}
public void testDanish() throws Exception {
Analyzer a = new DanishAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo.bar.FOO.BAR",
new String[] { "foo.bar.foo.bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" });
// Test stemming
testStemmer(a, "danish");
}
public void testDutch() throws Exception {
Analyzer a = new DutchAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo.bar.FOO.BAR",
new String[] { "foo.bar.foo.bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" });
// Test stemming
testStemmer(a, "dutch");
}
public void testEnglish() throws Exception {
Analyzer a = new EnglishAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo.bar.FOO.BAR",
new String[] { "foo.bar.foo.bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" words",
new String[] { "quot", "word" });
assertAnalyzesTo(a, "earring",
new String[] { "earring" });
// Test stemming
testStemmer(a, "english");
}
public void testFinnish() throws Exception {
Analyzer a = new FinnishAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo.bar.FOO.BAR",
new String[] { "foo.bar.foo.bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" });
// Test stemming
testStemmer(a, "finnish");
}
public void testFrench() throws Exception {
Analyzer a = new FrenchAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo.bar.FOO.BAR",
new String[] { "foo.bar.foo.bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" });
// Test stemming
testStemmer(a, "french");
}
public void testGerman() throws Exception {
Analyzer a = new GermanAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo.bar.FOO.BAR",
new String[] { "foo.bar.foo.bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" });
// Test stemming
testStemmer(a, "german");
}
public void testItalian() throws Exception {
Analyzer a = new ItalianAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" });
assertAnalyzesTo(a, "MANGIAR",
new String[] { "mang" });
assertAnalyzesTo(a, "MANGIARE",
new String[] { "mang" });
// Test stemming
testStemmer(a, "italian");
}
public void testNorwegian() throws Exception {
Analyzer a = new NorwegianAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" });
// Test stemming
testStemmer(a, "norwegian");
}
public void testPortuguese() throws Exception {
Analyzer a = new PortugueseAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" });
// Test stemming
testStemmer(a, "portuguese");
}
public void testRussian() throws Exception {
Analyzer a = new RussianAnalyzer();
/* assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo.bar.FOO.BAR",
new String[] { "foo.bar.foo.bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" }); */
// Test stemming using KOI8
testStemmer(a, "russian");
}
public void testSpanish() throws Exception {
Analyzer a = new SpanishAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quot", "word" });
// Test stemming
testStemmer(a, "spanish");
}
public void testSwedish() throws Exception {
Analyzer a = new SwedishAnalyzer();
assertAnalyzesTo(a, "foo bar FOO BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "foo bar . FOO <> BAR",
new String[] { "foo", "bar", "foo", "bar" });
assertAnalyzesTo(a, "U.S.A.",
new String[] { "usa" });
assertAnalyzesTo(a, "C++",
new String[] { "c" });
assertAnalyzesTo(a, "B2B",
new String[] { "b2b" });
assertAnalyzesTo(a, "2B",
new String[] { "2b" });
assertAnalyzesTo(a, "\"QUOTED\" word",
new String[] { "quoted", "word" });
// Test stemming
testStemmer(a, "swedish");
}
public void testStemmer(Analyzer anAnalyzer, String aLanguage) throws Exception {
String str;
String path = "c:\\projects\\cvs\\jakarta-lucene\\src\\java\\org\\apache\\lucene\\analysis\\snowball\\";
String suffixIn = ".in";
String suffixOut = ".out";
String suffixStopwords = ".stw";
// open and read words into an array list
ArrayList words = new ArrayList();
BufferedReader inWords =
new BufferedReader(
new InputStreamReader(
new FileInputStream(path + aLanguage + suffixIn)));
while ((str = inWords.readLine()) != null) {
words.add(str);
}
inWords.close();
// open and read stems into an array list
ArrayList stems = new ArrayList();
BufferedReader inStems =
new BufferedReader(
new InputStreamReader(
new FileInputStream(path + aLanguage + suffixOut)));
while ((str = inStems.readLine()) != null) {
stems.add(str);
}
inStems.close();
// open and read stopwords into a hash table
Hashtable stops = new Hashtable();
BufferedReader inStops =
new BufferedReader(
new InputStreamReader(
new FileInputStream(path + aLanguage + suffixStopwords)));
while ((str = inStops.readLine()) != null) {
stops.put(str, str);
}
inStops.close();
for (int i = 0; i < words.size(); i++)
if (!stops.containsKey(words.get(i))) {
TokenStream ts = anAnalyzer.tokenStream("dummy", new StringReader((String)words.get(i)));
Token t = ts.next();
assertEquals((String)stems.get(i), t.termText());
}
stops.clear();
words.clear();
stems.clear();
}
public static void main(String[] args) {
junit.textui.TestRunner.run(SnowballAnalyzerTest.class);
}
}
