"document 2 title</head><body>this is english</body></html>",
"<html><head><meta name=\"dc.language\" content=\"en\"><title>document 3 title</head><body>this is english</body></html>" };
<<<<<<< HEAD
// Tika does not return "fi" but null
=======
// known issue with attributed not being passed by Tika
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
String metalanguages[] = { "fi", "en", "en" };
/**</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> "<html><head><meta http-equiv=\"content-language\" content=\"en\"><title>document 2 title</head><body>this is english</body></html>",
"<html><head><meta name=\"dc.language\" content=\"en\"><title>document 3 title</head><body>this is english</body></html>" };
// known issue with attributed not being passed by Tika
String metalanguages[] = { "fi", "en", "en" };
/**</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestHTMLLanguageParser.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Comment</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>import junit.framework.TestSuite;
import junit.textui.TestRunner;
<<<<<<< HEAD
=======
import org.apache.lucene.analysis.Token;
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
import org.apache.nutch.analysis.lang.NGramProfile.NGramEntry;
import org.apache.nutch.util.NutchConfiguration;
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>import junit.framework.TestSuite;
import junit.textui.TestRunner;
import org.apache.lucene.analysis.Token;
import org.apache.nutch.analysis.lang.NGramProfile.NGramEntry;
import org.apache.nutch.util.NutchConfiguration;
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestLanguageIdentifier.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Import</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> */
public class TestLanguageIdentifier extends TestCase {
<<<<<<< HEAD
public TestLanguageIdentifier(String testName) {
super(testName);
}
public static Test suite() {
return new TestSuite(TestLanguageIdentifier.class);
}
public static void main(String[] args) {
TestRunner.run(suite());
}
String tokencontent1 = "testaddtoken";
String tokencontent2 = "anotherteststring";
int[] counts1 = { 3, 2, 2, 1, 1, 1, 1, 1 };
String[] chars1 = { "t", "d", "e", "a", "k", "n", "o", "s" };
/**
* Test addFromToken method
*
*/
public void testAddToken() {
NGramProfile p = new NGramProfile("test", 1, 1);
p.add(tokencontent1);
p.normalize();
testCounts(p.getSorted(), counts1);
testContents(p.getSorted(), chars1);
}
/**
* Test analyze method
*/
public void testAnalyze() {
String tokencontent = "testmeagain";
NGramProfile p = new NGramProfile("test", 1, 1);
p.analyze(new StringBuilder(tokencontent));
// test that profile size is ok, eg 9 different NGramEntries "tesmagin"
assertEquals(8, p.getSorted().size());
}
/**
* Test addNGrams method with StringBuffer argument
*
*/
public void testAddNGramsStringBuffer() {
String tokencontent = "testmeagain";
=======
public TestLanguageIdentifier(String testName) {
super(testName);
}
public static Test suite() {
return new TestSuite(TestLanguageIdentifier.class);
}
public static void main(String[] args) {
TestRunner.run(suite());
}
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
NGramProfile p = new NGramProfile("test", 1, 1);
p.add(new StringBuffer(tokencontent));</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> */
public class TestLanguageIdentifier extends TestCase {
public TestLanguageIdentifier(String testName) {
super(testName);
}
public static Test suite() {
return new TestSuite(TestLanguageIdentifier.class);
}
public static void main(String[] args) {
TestRunner.run(suite());
}
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestLanguageIdentifier.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Array initializer</td>
</tr>
<tr>
<td data-title="KindConflict"> Attribute</td>
</tr>
<tr>
<td data-title="KindConflict"> Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> Method declaration</td>
</tr>
<tr>
<td data-title="KindConflict"> Method signature</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>
}
<<<<<<< HEAD
/**
* test getSorted method
*/
public void testGetSorted() {
int[] count = { 4, 3, 1 };
String[] ngram = { "a", "b", "c" };
String teststring = "AAaaBbbC";
NGramProfile p = new NGramProfile("test", 1, 1);
p.analyze(new StringBuilder(teststring));
// test size of profile
assertEquals(3, p.getSorted().size());
testCounts(p.getSorted(), count);
testContents(p.getSorted(), ngram);
}
public void testGetSimilarity() {
NGramProfile a = new NGramProfile("a", 1, 1);
NGramProfile b = new NGramProfile("b", 1, 1);
a.analyze(new StringBuilder(tokencontent1));
b.analyze(new StringBuilder(tokencontent2));
// because of rounding errors might slightly return different results
assertEquals(a.getSimilarity(b), b.getSimilarity(a), 0.0000002);
}
public void testExactMatch() {
NGramProfile a = new NGramProfile("a", 1, 1);
a.analyze(new StringBuilder(tokencontent1));
assertEquals(a.getSimilarity(a), 0, 0);
}
public void testIO() {
// Create profile and set some contents
NGramProfile a = new NGramProfile("a", 1, 1);
a.analyze(new StringBuilder(this.tokencontent1));
NGramProfile b = new NGramProfile("a_from_inputstream", 1, 1);
// save profile
ByteArrayOutputStream os = new ByteArrayOutputStream();
try {
a.save(os);
os.close();
} catch (Exception e) {
fail();
}
// load profile
InputStream is = new ByteArrayInputStream(os.toByteArray());
try {
b.load(is);
is.close();
} catch (Exception e) {
fail();
}
// check it
testCounts(b.getSorted(), counts1);
testContents(b.getSorted(), chars1);
}
private void testContents(List<NGramEntry> entries, String contents[]) {
int c = 0;
for (NGramEntry nge : entries) {
assertEquals(contents[c], nge.getSeq().toString());
c++;
}
}
private void testCounts(List<NGramEntry> entries, int counts[]) {
int c = 0;
for (NGramEntry nge : entries) {
System.out.println(nge);
assertEquals(counts[c], nge.getCount());
c++;
}
}
public void testIdentify() {
try {
long total = 0;
LanguageIdentifier idfr = new LanguageIdentifier(
NutchConfiguration.create());
BufferedReader in = new BufferedReader(new InputStreamReader(this
.getClass().getResourceAsStream("test-referencial.txt")));
String line = null;
while ((line = in.readLine()) != null) {
String[] tokens = line.split(";");
if (!tokens[0].equals("")) {
long start = System.currentTimeMillis();
// Identify the whole file
String lang = idfr.identify(this.getClass()
.getResourceAsStream(tokens[0]), "UTF-8");
total += System.currentTimeMillis() - start;
assertEquals(tokens[1], lang);
// Then, each line of the file...
BufferedReader testFile = new BufferedReader(
new InputStreamReader(this.getClass()
.getResourceAsStream(tokens[0]), "UTF-8"));
String testLine = null;
while ((testLine = testFile.readLine()) != null) {
testLine = testLine.trim();
if (testLine.length() > 256) {
lang = idfr.identify(testLine);
assertEquals(tokens[1], lang);
}
}
testFile.close();
}
}
in.close();
System.out.println("Total Time=" + total);
} catch (Exception e) {
e.printStackTrace();
fail(e.toString());
}
}
=======
/**
* Test addFromToken method
*
*/
public void testAddToken() {
NGramProfile p = new NGramProfile("test", 1, 1);
Token t = new Token(tokencontent1, 0, tokencontent1.length());
p.add(t);
p.normalize();
testCounts(p.getSorted(), counts1);
testContents(p.getSorted(), chars1);
}
/**
* Test analyze method
*/
public void testAnalyze() {
String tokencontent = "testmeagain";
NGramProfile p = new NGramProfile("test", 1, 1);
p.analyze(new StringBuilder(tokencontent));
// test that profile size is ok, eg 9 different NGramEntries "tesmagin"
assertEquals(8, p.getSorted().size());
}
/**
* Test addNGrams method with StringBuffer argument
*
*/
public void testAddNGramsStringBuffer() {
String tokencontent = "testmeagain";
NGramProfile p = new NGramProfile("test", 1, 1);
p.add(new StringBuffer(tokencontent));
// test that profile size is ok, eg 8 different NGramEntries "tesmagin"
assertEquals(8, p.getSorted().size());
}
/**
* test getSorted method
*/
public void testGetSorted() {
int[] count = { 4, 3, 1 };
String[] ngram = { "a", "b", "c" };
String teststring = "AAaaBbbC";
NGramProfile p = new NGramProfile("test", 1, 1);
p.analyze(new StringBuilder(teststring));
// test size of profile
assertEquals(3, p.getSorted().size());
testCounts(p.getSorted(), count);
testContents(p.getSorted(), ngram);
}
public void testGetSimilarity() {
NGramProfile a = new NGramProfile("a", 1, 1);
NGramProfile b = new NGramProfile("b", 1, 1);
a.analyze(new StringBuilder(tokencontent1));
b.analyze(new StringBuilder(tokencontent2));
// because of rounding errors might slightly return different results
assertEquals(a.getSimilarity(b), b.getSimilarity(a), 0.0000002);
}
public void testExactMatch() {
NGramProfile a = new NGramProfile("a", 1, 1);
a.analyze(new StringBuilder(tokencontent1));
assertEquals(a.getSimilarity(a), 0, 0);
}
public void testIO() {
// Create profile and set some contents
NGramProfile a = new NGramProfile("a", 1, 1);
a.analyze(new StringBuilder(this.tokencontent1));
NGramProfile b = new NGramProfile("a_from_inputstream", 1, 1);
// save profile
ByteArrayOutputStream os = new ByteArrayOutputStream();
try {
a.save(os);
os.close();
} catch (Exception e) {
fail();
}
// load profile
InputStream is = new ByteArrayInputStream(os.toByteArray());
try {
b.load(is);
is.close();
} catch (Exception e) {
fail();
}
// check it
testCounts(b.getSorted(), counts1);
testContents(b.getSorted(), chars1);
}
private void testContents(List<NGramEntry> entries, String contents[]) {
int c = 0;
for (NGramEntry nge : entries) {
assertEquals(contents[c], nge.getSeq().toString());
c++;
}
}
private void testCounts(List<NGramEntry> entries, int counts[]) {
int c = 0;
for (NGramEntry nge : entries) {
System.out.println(nge);
assertEquals(counts[c], nge.getCount());
c++;
}
}
public void testIdentify() {
try {
long total = 0;
LanguageIdentifier idfr = new LanguageIdentifier(NutchConfiguration
.create());
BufferedReader in = new BufferedReader(new InputStreamReader(this
.getClass().getResourceAsStream("test-referencial.txt")));
String line = null;
while ((line = in.readLine()) != null) {
String[] tokens = line.split(";");
if (!tokens[0].equals("")) {
long start = System.currentTimeMillis();
// Identify the whole file
String lang = idfr.identify(this.getClass().getResourceAsStream(
tokens[0]), "UTF-8");
total += System.currentTimeMillis() - start;
assertEquals(tokens[1], lang);
// Then, each line of the file...
BufferedReader testFile = new BufferedReader(new InputStreamReader(
this.getClass().getResourceAsStream(tokens[0]), "UTF-8"));
String testLine = null;
while ((testLine = testFile.readLine()) != null) {
testLine = testLine.trim();
if (testLine.length() > 256) {
lang = idfr.identify(testLine);
assertEquals(tokens[1], lang);
}
}
testFile.close();
}
}
in.close();
System.out.println("Total Time=" + total);
} catch (Exception e) {
e.printStackTrace();
fail(e.toString());
}
}
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>
String[] chars1 = { "t", "d", "e", "a", "k", "n", "o", "s" };
/**
* Test addFromToken method
*
*/
public void testAddToken() {
NGramProfile p = new NGramProfile("test", 1, 1);
Token t = new Token(tokencontent1, 0, tokencontent1.length());
p.add(t);
p.normalize();
testCounts(p.getSorted(), counts1);
testContents(p.getSorted(), chars1);
}
/**
* Test analyze method
*/
public void testAnalyze() {
String tokencontent = "testmeagain";
NGramProfile p = new NGramProfile("test", 1, 1);
p.analyze(new StringBuilder(tokencontent));
// test that profile size is ok, eg 9 different NGramEntries "tesmagin"
assertEquals(8, p.getSorted().size());
}
/**
* Test addNGrams method with StringBuffer argument
*
*/
public void testAddNGramsStringBuffer() {
String tokencontent = "testmeagain";
NGramProfile p = new NGramProfile("test", 1, 1);
p.add(new StringBuffer(tokencontent));
// test that profile size is ok, eg 8 different NGramEntries "tesmagin"
assertEquals(8, p.getSorted().size());
}
/**
* test getSorted method
*/
public void testGetSorted() {
int[] count = { 4, 3, 1 };
String[] ngram = { "a", "b", "c" };
String teststring = "AAaaBbbC";
NGramProfile p = new NGramProfile("test", 1, 1);
p.analyze(new StringBuilder(teststring));
// test size of profile
assertEquals(3, p.getSorted().size());
testCounts(p.getSorted(), count);
testContents(p.getSorted(), ngram);
}
public void testGetSimilarity() {
NGramProfile a = new NGramProfile("a", 1, 1);
NGramProfile b = new NGramProfile("b", 1, 1);
a.analyze(new StringBuilder(tokencontent1));
b.analyze(new StringBuilder(tokencontent2));
// because of rounding errors might slightly return different results
assertEquals(a.getSimilarity(b), b.getSimilarity(a), 0.0000002);
}
public void testExactMatch() {
NGramProfile a = new NGramProfile("a", 1, 1);
a.analyze(new StringBuilder(tokencontent1));
assertEquals(a.getSimilarity(a), 0, 0);
}
public void testIO() {
// Create profile and set some contents
NGramProfile a = new NGramProfile("a", 1, 1);
a.analyze(new StringBuilder(this.tokencontent1));
NGramProfile b = new NGramProfile("a_from_inputstream", 1, 1);
// save profile
ByteArrayOutputStream os = new ByteArrayOutputStream();
try {
a.save(os);
os.close();
} catch (Exception e) {
fail();
}
// load profile
InputStream is = new ByteArrayInputStream(os.toByteArray());
try {
b.load(is);
is.close();
} catch (Exception e) {
fail();
}
// check it
testCounts(b.getSorted(), counts1);
testContents(b.getSorted(), chars1);
}
private void testContents(List<NGramEntry> entries, String contents[]) {
int c = 0;
for (NGramEntry nge : entries) {
assertEquals(contents[c], nge.getSeq().toString());
c++;
}
}
private void testCounts(List<NGramEntry> entries, int counts[]) {
int c = 0;
for (NGramEntry nge : entries) {
System.out.println(nge);
assertEquals(counts[c], nge.getCount());
c++;
}
}
public void testIdentify() {
try {
long total = 0;
LanguageIdentifier idfr = new LanguageIdentifier(NutchConfiguration
.create());
BufferedReader in = new BufferedReader(new InputStreamReader(this
.getClass().getResourceAsStream("test-referencial.txt")));
String line = null;
while ((line = in.readLine()) != null) {
String[] tokens = line.split(";");
if (!tokens[0].equals("")) {
long start = System.currentTimeMillis();
// Identify the whole file
String lang = idfr.identify(this.getClass().getResourceAsStream(
tokens[0]), "UTF-8");
total += System.currentTimeMillis() - start;
assertEquals(tokens[1], lang);
// Then, each line of the file...
BufferedReader testFile = new BufferedReader(new InputStreamReader(
this.getClass().getResourceAsStream(tokens[0]), "UTF-8"));
String testLine = null;
while ((testLine = testFile.readLine()) != null) {
testLine = testLine.trim();
if (testLine.length() > 256) {
lang = idfr.identify(testLine);
assertEquals(tokens[1], lang);
}
}
testFile.close();
}
}
in.close();
System.out.println("Total Time=" + total);
} catch (Exception e) {
e.printStackTrace();
fail(e.toString());
}
}
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestLanguageIdentifier.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> Method declaration</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> }
robots = new RobotRulesParser();
}
<<<<<<< HEAD
// Inherited Javadoc
public void setConf(Configuration conf) {
this.conf = conf;
this.proxyHost = conf.get("http.proxy.host");
this.proxyPort = conf.getInt("http.proxy.port", 8080);
this.useProxy = (proxyHost != null && proxyHost.length() > 0);
this.timeout = conf.getInt("http.timeout", 10000);
this.maxContent = conf.getInt("http.content.limit", 64 * 1024);
this.maxDelays = conf.getInt("http.max.delays", 3);
this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1);
this.userAgent = getAgentString(conf.get("http.agent.name"), conf.get("http.agent.version"), conf
.get("http.agent.description"), conf.get("http.agent.url"), conf.get("http.agent.email"));
this.acceptLanguage = conf.get("http.accept.language", acceptLanguage);
this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000);
this.maxCrawlDelay = (long)(conf.getInt("fetcher.max.crawl.delay", -1) * 1000);
// backward-compatible default setting
this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", true);
this.useHttp11 = conf.getBoolean("http.useHttp11", false);
this.robots.setConf(conf);
this.checkBlocking = conf.getBoolean(Protocol.CHECK_BLOCKING, true);
this.checkRobots = conf.getBoolean(Protocol.CHECK_ROBOTS, true);
logConf();
}
=======
// Inherited Javadoc
public void setConf(Configuration conf) {
this.conf = conf;
this.proxyHost = conf.get("http.proxy.host");
this.proxyPort = conf.getInt("http.proxy.port", 8080);
this.useProxy = (proxyHost != null && proxyHost.length() > 0);
this.timeout = conf.getInt("http.timeout", 10000);
this.maxContent = conf.getInt("http.content.limit", 64 * 1024);
this.maxDelays = conf.getInt("http.max.delays", 3);
this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1);
this.userAgent = getAgentString(conf.get("http.agent.name"), conf.get("http.agent.version"), conf
.get("http.agent.description"), conf.get("http.agent.url"), conf.get("http.agent.email"));
this.acceptLanguage = conf.get("http.accept.language", acceptLanguage);
this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000);
this.maxCrawlDelay = (conf.getInt("fetcher.max.crawl.delay", -1) * 1000);
// backward-compatible default setting
this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", true);
this.mimeTypes = new MimeUtil(conf);
this.useHttp11 = conf.getBoolean("http.useHttp11", false);
this.robots.setConf(conf);
this.checkBlocking = conf.getBoolean(Protocol.CHECK_BLOCKING, true);
this.checkRobots = conf.getBoolean(Protocol.CHECK_ROBOTS, true);
logConf();
}
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
// Inherited Javadoc
public Configuration getConf() {</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> }
robots = new RobotRulesParser();
}
// Inherited Javadoc
public void setConf(Configuration conf) {
this.conf = conf;
this.proxyHost = conf.get("http.proxy.host");
this.proxyPort = conf.getInt("http.proxy.port", 8080);
this.useProxy = (proxyHost != null && proxyHost.length() > 0);
this.timeout = conf.getInt("http.timeout", 10000);
this.maxContent = conf.getInt("http.content.limit", 64 * 1024);
this.maxDelays = conf.getInt("http.max.delays", 3);
this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1);
this.userAgent = getAgentString(conf.get("http.agent.name"), conf.get("http.agent.version"), conf
.get("http.agent.description"), conf.get("http.agent.url"), conf.get("http.agent.email"));
this.acceptLanguage = conf.get("http.accept.language", acceptLanguage);
this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000);
this.maxCrawlDelay = (conf.getInt("fetcher.max.crawl.delay", -1) * 1000);
// backward-compatible default setting
this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", true);
this.mimeTypes = new MimeUtil(conf);
this.useHttp11 = conf.getBoolean("http.useHttp11", false);
this.robots.setConf(conf);
this.checkBlocking = conf.getBoolean(Protocol.CHECK_BLOCKING, true);
this.checkRobots = conf.getBoolean(Protocol.CHECK_ROBOTS, true);
logConf();
}
// Inherited Javadoc
public Configuration getConf() {</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">HttpBase.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> Method declaration</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> public ProtocolOutput getProtocolOutput(String url, WebPage page) {
try {
<<<<<<< HEAD
URL u = new URL(urlString);
long delay = serverDelay;
=======
URL u = new URL(url);
long delay = serverDelay;
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
if (checkRobots) {
try {
if (!robots.isAllowed(this, u)) {</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> public ProtocolOutput getProtocolOutput(String url, WebPage page) {
try {
URL u = new URL(url);
long delay = serverDelay;
if (checkRobots) {
try {
if (!robots.isAllowed(this, u)) {</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">HttpBase.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Attribute</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> if (LOG.isTraceEnabled()) { LOG.trace("cache miss " + url); }
try {
Response response = http.getResponse(new URL(url, "/robots.txt"),
<<<<<<< HEAD
new CrawlDatum(), true);
=======
new WebPage(), true);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
// try one level of redirection ?
if (response.getCode() == 301 || response.getCode() == 302) {
String redirection = response.getHeader("Location");</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> if (LOG.isTraceEnabled()) { LOG.trace("cache miss " + url); }
try {
Response response = http.getResponse(new URL(url, "/robots.txt"),
new WebPage(), true);
// try one level of redirection ?
if (response.getCode() == 301 || response.getCode() == 302) {
String redirection = response.getHeader("Location");</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">RobotRulesParser.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> } else {
redir = new URL(redirection);
}
<<<<<<< HEAD
response = http.getResponse(redir, new CrawlDatum(), true);
=======
response = http.getResponse(redir, new WebPage(), true);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}
}
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> } else {
redir = new URL(redirection);
}
response = http.getResponse(redir, new WebPage(), true);
}
}
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">RobotRulesParser.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>package org.apache.nutch.microformats.reltag;
// Nutch imports
<<<<<<< HEAD
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.hadoop.io.Text;
import org.apache.nutch.parse.Parse;
=======
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.HashSet;
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>package org.apache.nutch.microformats.reltag;
// Nutch imports
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.HashSet;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">RelTagIndexingFilter.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Import</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> return this.conf;
}
<<<<<<< HEAD
/* ----------------------------- *
* <implementation:Configurable> *
* ----------------------------- */
public void setConf(Configuration conf) {
this.conf = conf;
}
=======
@Override
public NutchDocument filter(NutchDocument doc, String url, WebPage page)
throws IndexingException {
// Check if some Rel-Tags found, possibly put there by RelTagParser
ByteBuffer bb = page.getFromMetadata(new Utf8(RelTagParser.REL_TAG));
if (bb != null) {
String[] tags = new String(bb.array()).split("\t");
for (int i = 0; i < tags.length; i++) {
doc.add("tag", tags[i]);
}
}
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
return doc;
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> return this.conf;
}
@Override
public NutchDocument filter(NutchDocument doc, String url, WebPage page)
throws IndexingException {
// Check if some Rel-Tags found, possibly put there by RelTagParser
ByteBuffer bb = page.getFromMetadata(new Utf8(RelTagParser.REL_TAG));
if (bb != null) {
String[] tags = new String(bb.array()).split("\t");
for (int i = 0; i < tags.length; i++) {
doc.add("tag", tags[i]);
}
}
return doc;
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">RelTagIndexingFilter.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Annotation</td>
</tr>
<tr>
<td data-title="KindConflict"> Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> If statement</td>
</tr>
<tr>
<td data-title="KindConflict"> Method declaration</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Method signature</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> testDOMs[i]= node;
}
try {
<<<<<<< HEAD
answerOutlinks = new Outlink[][]{
{
new Outlink("http://www.nutch.org", "anchor"),
},
{
new Outlink("http://www.nutch.org/", "home"),
new Outlink("http://www.nutch.org/docs/bot.html", "bots"),
},
{
new Outlink("http://www.nutch.org/", "separate this"),
new Outlink("http://www.nutch.org/docs/ok", "from this"),
},
{
new Outlink("http://www.nutch.org/", "home"),
new Outlink("http://www.nutch.org/docs/1", "1"),
new Outlink("http://www.nutch.org/docs/2", "2"),
},
{
new Outlink("http://www.nutch.org/frames/top.html", ""),
new Outlink("http://www.nutch.org/frames/left.html", ""),
new Outlink("http://www.nutch.org/frames/invalid.html", ""),
new Outlink("http://www.nutch.org/frames/right.html", ""),
},
{
new Outlink("http://www.nutch.org/maps/logo.gif", ""),
new Outlink("http://www.nutch.org/index.html", ""),
new Outlink("http://www.nutch.org/maps/#bottom", ""),
new Outlink("http://www.nutch.org/bot.html", ""),
new Outlink("http://www.nutch.org/docs/index.html", ""),
},
{
new Outlink("http://www.nutch.org/index.html", "whitespace test"),
},
{
},
{
new Outlink("http://www.nutch.org/dummy.jsp", "test2"),
},
{
},
{
new Outlink("http://www.nutch.org/;x", "anchor1"),
new Outlink("http://www.nutch.org/g;x", "anchor2"),
new Outlink("http://www.nutch.org/g;x?y#s", "anchor3")
},
{
new Outlink("http://www.nutch.org/g;something", "anchor1"),
new Outlink("http://www.nutch.org/g;something?y#s", "anchor2"),
new Outlink("http://www.nutch.org/;something?y=1", "anchor3"),
new Outlink("http://www.nutch.org/;something?y=1#s", "anchor4"),
new Outlink("http://www.nutch.org/?y=1;somethingelse", "anchor5")
},
{
new Outlink("http://www.nutch.org/g", ""),
new Outlink("http://www.nutch.org/g1", ""),
new Outlink("http://www.nutch.org/g2", "bla bla"),
new Outlink("http://www.nutch.org/test.gif", "bla bla"),
}
};
=======
answerOutlinks = new Outlink[][]{
{
new Outlink("http://www.nutch.org", "anchor"),
},
{
new Outlink("http://www.nutch.org/", "home"),
new Outlink("http://www.nutch.org/docs/bot.html", "bots"),
},
{
new Outlink("http://www.nutch.org/", "separate this"),
new Outlink("http://www.nutch.org/docs/ok", "from this"),
},
{
new Outlink("http://www.nutch.org/", "home"),
new Outlink("http://www.nutch.org/docs/1", "1"),
new Outlink("http://www.nutch.org/docs/2", "2"),
},
{
new Outlink("http://www.nutch.org/frames/top.html", ""),
new Outlink("http://www.nutch.org/frames/left.html", ""),
new Outlink("http://www.nutch.org/frames/invalid.html", ""),
new Outlink("http://www.nutch.org/frames/right.html", ""),
},
{
new Outlink("http://www.nutch.org/maps/logo.gif", ""),
new Outlink("http://www.nutch.org/index.html", ""),
new Outlink("http://www.nutch.org/maps/#bottom", ""),
new Outlink("http://www.nutch.org/bot.html", ""),
new Outlink("http://www.nutch.org/docs/index.html", ""),
},
{
new Outlink("http://www.nutch.org/index.html", "whitespace test"),
},
{
},
{
new Outlink("http://www.nutch.org/dummy.jsp", "test2"),
},
{
},
{
new Outlink("http://www.nutch.org/;x", "anchor1"),
new Outlink("http://www.nutch.org/g;x", "anchor2"),
new Outlink("http://www.nutch.org/g;x?y#s", "anchor3")
},
{
new Outlink("http://www.nutch.org/g;something", "anchor1"),
new Outlink("http://www.nutch.org/g;something?y#s", "anchor2"),
new Outlink("http://www.nutch.org/;something?y=1", "anchor3"),
new Outlink("http://www.nutch.org/;something?y=1#s", "anchor4"),
new Outlink("http://www.nutch.org/?y=1;somethingelse", "anchor5")
}
};
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
} catch (MalformedURLException e) {
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> testDOMs[i]= node;
}
try {
answerOutlinks = new Outlink[][]{
{
new Outlink("http://www.nutch.org", "anchor"),
},
{
new Outlink("http://www.nutch.org/", "home"),
new Outlink("http://www.nutch.org/docs/bot.html", "bots"),
},
{
new Outlink("http://www.nutch.org/", "separate this"),
new Outlink("http://www.nutch.org/docs/ok", "from this"),
},
{
new Outlink("http://www.nutch.org/", "home"),
new Outlink("http://www.nutch.org/docs/1", "1"),
new Outlink("http://www.nutch.org/docs/2", "2"),
},
{
new Outlink("http://www.nutch.org/frames/top.html", ""),
new Outlink("http://www.nutch.org/frames/left.html", ""),
new Outlink("http://www.nutch.org/frames/invalid.html", ""),
new Outlink("http://www.nutch.org/frames/right.html", ""),
},
{
new Outlink("http://www.nutch.org/maps/logo.gif", ""),
new Outlink("http://www.nutch.org/index.html", ""),
new Outlink("http://www.nutch.org/maps/#bottom", ""),
new Outlink("http://www.nutch.org/bot.html", ""),
new Outlink("http://www.nutch.org/docs/index.html", ""),
},
{
new Outlink("http://www.nutch.org/index.html", "whitespace test"),
},
{
},
{
new Outlink("http://www.nutch.org/dummy.jsp", "test2"),
},
{
},
{
new Outlink("http://www.nutch.org/;x", "anchor1"),
new Outlink("http://www.nutch.org/g;x", "anchor2"),
new Outlink("http://www.nutch.org/g;x?y#s", "anchor3")
},
{
new Outlink("http://www.nutch.org/g;something", "anchor1"),
new Outlink("http://www.nutch.org/g;something?y#s", "anchor2"),
new Outlink("http://www.nutch.org/;something?y=1", "anchor3"),
new Outlink("http://www.nutch.org/;something?y=1#s", "anchor4"),
new Outlink("http://www.nutch.org/?y=1;somethingelse", "anchor5")
}
};
} catch (MalformedURLException e) {
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestDOMContentUtils.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Attribute</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> reqStr.append(userAgent);
reqStr.append("\r\n");
}
<<<<<<< HEAD
reqStr.append("Accept-Language: ");
reqStr.append(this.http.getAcceptLanguage());
reqStr.append("\r\n");
if (datum.getModifiedTime() > 0) {
reqStr.append("If-Modified-Since: " + HttpDateFormat.toString(datum.getModifiedTime()));
reqStr.append("\r\n");
}
reqStr.append("\r\n");
=======
if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) {
reqStr.append("If-Modified-Since: " +
HttpDateFormat.toString(page.getModifiedTime()));
reqStr.append("\r\n");
}
reqStr.append("\r\n");
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
byte[] reqBytes= reqStr.toString().getBytes();
req.write(reqBytes);</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> reqStr.append(userAgent);
reqStr.append("\r\n");
}
if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) {
reqStr.append("If-Modified-Since: " +
HttpDateFormat.toString(page.getModifiedTime()));
reqStr.append("\r\n");
}
reqStr.append("\r\n");
byte[] reqBytes= reqStr.toString().getBytes();
req.write(reqBytes);</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">HttpResponse.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">If statement</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>
<<<<<<< HEAD
public class Http extends HttpBase {
*/
public static final Log LOG = LogFactory.getLog(Http.class);
private static MultiThreadedHttpConnectionManager connectionManager =
new MultiThreadedHttpConnectionManager();
// Since the Configuration has not yet been set,
// then an unconfigured client is returned.
private static HttpClient client = new HttpClient(connectionManager);
private static String defaultUsername;
private static String defaultPassword;
private static String defaultRealm;
private static String defaultScheme;
private static String authFile;
private static String agentHost;
private static boolean authRulesRead = false;
private static Configuration conf;
int maxThreadsTotal = 10;
private String proxyUsername;
private String proxyPassword;
private String proxyRealm;
* @param realm
/**
* Returns the configured HTTP client.
*
* <code>scheme</code>.
* @return HTTP client
*/
static synchronized HttpClient getClient() {
return client;
}
/**
* Constructs this plugin.
*/
public Http() {
super(LOG);
}
/**
* Reads the configuration from the Nutch configuration files and sets
* the configuration.
*
* @param conf Configuration
*/
public void setConf(Configuration conf) {
super.setConf(conf);
this.conf = conf;
this.maxThreadsTotal = conf.getInt("fetcher.threads.fetch", 10);
this.proxyUsername = conf.get("http.proxy.username", "");
this.proxyPassword = conf.get("http.proxy.password", "");
this.proxyRealm = conf.get("http.proxy.realm", "");
agentHost = conf.get("http.agent.host", "");
authFile = conf.get("http.auth.file", "");
configureClient();
try {
setCredentials();
} catch (Exception ex) {
if (LOG.isFatalEnabled()) {
LOG.fatal("Could not read " + authFile + " : " + ex.getMessage());
ex.printStackTrace(LogUtil.getErrorStream(LOG));
}
}
}
/**
* Main method.
*
* @param args Command line arguments
*/
public static void main(String[] args) throws Exception {
Http http = new Http();
http.setConf(NutchConfiguration.create());
main(http, args);
}
/**
* Fetches the <code>url</code> with a configured HTTP client and
* gets the response.
*
* @param url URL to be fetched
* @param datum Crawl data
* @param redirect Follow redirects if and only if true
* @return HTTP response
*/
protected Response getResponse(URL url, CrawlDatum datum, boolean redirect)
throws ProtocolException, IOException {
resolveCredentials(url);
return new HttpResponse(this, url, datum, redirect);
}
/**
* Configures the HTTP client
*/
private void configureClient() {
// Set up an HTTPS socket factory that accepts self-signed certs.
Protocol https = new Protocol("https",
new DummySSLProtocolSocketFactory(), 443);
Protocol.registerProtocol("https", https);
HttpConnectionManagerParams params = connectionManager.getParams();
params.setConnectionTimeout(timeout);
params.setSoTimeout(timeout);
params.setSendBufferSize(BUFFER_SIZE);
params.setReceiveBufferSize(BUFFER_SIZE);
params.setMaxTotalConnections(maxThreadsTotal);
if (maxThreadsTotal > maxThreadsPerHost) {
params.setDefaultMaxConnectionsPerHost(maxThreadsPerHost);
} else {
params.setDefaultMaxConnectionsPerHost(maxThreadsTotal);
}
// executeMethod(HttpMethod) seems to ignore the connection timeout on the connection manager.
// set it explicitly on the HttpClient.
client.getParams().setConnectionManagerTimeout(timeout);
HostConfiguration hostConf = client.getHostConfiguration();
ArrayList headers = new ArrayList();
// Set the User Agent in the header
headers.add(new Header("User-Agent", userAgent));
// prefer English
headers.add(new Header("Accept-Language", acceptLanguage));
// prefer UTF-8
headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7"));
// prefer understandable formats
headers.add(new Header("Accept",
"text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
// accept gzipped content
+ url.getHost() + "; port: " + port
headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate"));
hostConf.getParams().setParameter("http.default-headers", headers);
// HTTP proxy server details
if (useProxy) {
hostConf.setProxy(proxyHost, proxyPort);
if (proxyUsername.length() > 0) {
AuthScope proxyAuthScope = getAuthScope(
this.proxyHost, this.proxyPort, this.proxyRealm);
NTCredentials proxyCredentials = new NTCredentials(
this.proxyUsername, this.proxyPassword,
this.agentHost, this.proxyRealm);
client.getState().setProxyCredentials(
proxyAuthScope, proxyCredentials);
}
}
}
/**
* Reads authentication configuration file (defined as
* 'http.auth.file' in Nutch configuration file) and sets the
* credentials for the configured authentication scopes in the HTTP
* client object.
*
* @throws ParserConfigurationException If a document builder can not
* be created.
* @throws SAXException If any parsing error occurs.
* @throws IOException If any I/O error occurs.
*/
private static synchronized void setCredentials() throws
ParserConfigurationException, SAXException, IOException {
if (authRulesRead)
return;
authRulesRead = true; // Avoid re-attempting to read
InputStream is = conf.getConfResourceAsInputStream(authFile);
if (is != null) {
Document doc = DocumentBuilderFactory.newInstance()
.newDocumentBuilder().parse(is);
Element rootElement = doc.getDocumentElement();
if (!"auth-configuration".equals(rootElement.getTagName())) {
if (LOG.isWarnEnabled())
LOG.warn("Bad auth conf file: root element <"
+ rootElement.getTagName() + "> found in " + authFile
+ " - must be <auth-configuration>");
}
// For each set of credentials
NodeList credList = rootElement.getChildNodes();
for (int i = 0; i < credList.getLength(); i++) {
Node credNode = credList.item(i);
if (!(credNode instanceof Element))
continue;
Element credElement = (Element) credNode;
if (!"credentials".equals(credElement.getTagName())) {
if (LOG.isWarnEnabled())
LOG.warn("Bad auth conf file: Element <"
+ credElement.getTagName() + "> not recognized in "
+ authFile + " - expected <credentials>");
continue;
}
String username = credElement.getAttribute("username");
String password = credElement.getAttribute("password");
// For each authentication scope
NodeList scopeList = credElement.getChildNodes();
for (int j = 0; j < scopeList.getLength(); j++) {
Node scopeNode = scopeList.item(j);
if (!(scopeNode instanceof Element))
continue;
Element scopeElement = (Element) scopeNode;
if ("default".equals(scopeElement.getTagName())) {
// Determine realm and scheme, if any
String realm = scopeElement.getAttribute("realm");
String scheme = scopeElement.getAttribute("scheme");
// Set default credentials
defaultUsername = username;
defaultPassword = password;
defaultRealm = realm;
defaultScheme = scheme;
if (LOG.isTraceEnabled()) {
*
* @param port
LOG.trace("Credentials - username: " + username
+ "; set as default"
+ " for realm: " + realm + "; scheme: " + scheme);
}
} else if ("authscope".equals(scopeElement.getTagName())) {
// Determine authentication scope details
String host = scopeElement.getAttribute("host");
int port = -1; // For setting port to AuthScope.ANY_PORT
try {
port = Integer.parseInt(
scopeElement.getAttribute("port"));
} catch (Exception ex) {
// do nothing, port is already set to any port
}
String realm = scopeElement.getAttribute("realm");
String scheme = scopeElement.getAttribute("scheme");
// Set credentials for the determined scope
AuthScope authScope = getAuthScope(host, port, realm, scheme);
NTCredentials credentials = new NTCredentials(
username, password, agentHost, realm);
client.getState().setCredentials(authScope, credentials);
if (LOG.isTraceEnabled()) {
LOG.trace("Credentials - username: " + username
+ "; set for AuthScope - " + "host: " + host
+ "; port: " + port + "; realm: " + realm
+ "; scheme: " + scheme);
}
} else {
if (LOG.isWarnEnabled())
LOG.warn("Bad auth conf file: Element <"
+ scopeElement.getTagName() + "> not recognized in "
+ authFile + " - expected <authscope>");
}
}
is.close();
}
}
}
/**
* If credentials for the authentication scope determined from the
* specified <code>url</code> is not already set in the HTTP client,
* then this method sets the default credentials to fetch the
* specified <code>url</code>. If credentials are found for the
* authentication scope, the method returns without altering the
* client.
*
* @param url URL to be fetched
*/
private void resolveCredentials(URL url) {
if (defaultUsername != null && defaultUsername.length() > 0) {
int port = url.getPort();
if (port == -1) {
if ("https".equals(url.getProtocol()))
port = 443;
else
port = 80;
}
AuthScope scope = new AuthScope(url.getHost(), port);
if (client.getState().getCredentials(scope) != null) {
if (LOG.isTraceEnabled())
LOG.trace("Pre-configured credentials with scope - host: "
+ "; found for url: " + url);
// Credentials are already configured, so do nothing and return
return;
}
if (LOG.isTraceEnabled())
LOG.trace("Pre-configured credentials with scope - host: "
+ url.getHost() + "; port: " + port
+ "; not found for url: " + url);
AuthScope serverAuthScope = getAuthScope(
url.getHost(), port, defaultRealm, defaultScheme);
NTCredentials serverCredentials = new NTCredentials(
defaultUsername, defaultPassword,
agentHost, defaultRealm);
client.getState().setCredentials(
serverAuthScope, serverCredentials);
}
}
/**
* Returns an authentication scope for the specified
* <code>host</code>, <code>port</code>, <code>realm</code> and
* @param host Host name or address.
* @param port Port number.
* @param realm Authentication realm.
* @param scheme Authentication scheme.
*/
private static AuthScope getAuthScope(String host, int port,
String realm, String scheme) {
if (host.length() == 0)
host = null;
if (port < 0)
port = -1;
if (realm.length() == 0)
realm = null;
if (scheme.length() == 0)
scheme = null;
return new AuthScope(host, port, realm, scheme);
}
/**
* Returns an authentication scope for the specified
* <code>host</code>, <code>port</code> and <code>realm</code>.
*
* @param host Host name or address.
* @param port Port number.
* @param realm Authentication realm.
*/
private static AuthScope getAuthScope(String host, int port,
String realm) {
return getAuthScope(host, port, realm, "");
}
}
=======
public static final Log LOG = LogFactory.getLog(Http.class);
private static MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
// Since the Configuration has not yet been set,
// then an unconfigured client is returned.
private static HttpClient client = new HttpClient(connectionManager);
private static String defaultUsername;
private static String defaultPassword;
private static String defaultRealm;
private static String defaultScheme;
private static String authFile;
private static String agentHost;
private static boolean authRulesRead = false;
private static Configuration conf;
int maxThreadsTotal = 10;
private String proxyUsername;
private String proxyPassword;
private String proxyRealm;
private static final Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
static {
FIELDS.add(WebPage.Field.MODIFIED_TIME);
FIELDS.add(WebPage.Field.HEADERS);
}
@Override
public Collection<Field> getFields() {
return FIELDS;
}
/**
* Returns the configured HTTP client.
*
* @return HTTP client
*/
static synchronized HttpClient getClient() {
return client;
}
/**
* Constructs this plugin.
*/
public Http() {
super(LOG);
}
/**
* Reads the configuration from the Nutch configuration files and sets the
* configuration.
*
* @param conf
* Configuration
*/
public void setConf(Configuration conf) {
super.setConf(conf);
this.conf = conf;
this.maxThreadsTotal = conf.getInt("fetcher.threads.fetch", 10);
this.proxyUsername = conf.get("http.proxy.username", "");
this.proxyPassword = conf.get("http.proxy.password", "");
this.proxyRealm = conf.get("http.proxy.realm", "");
agentHost = conf.get("http.agent.host", "");
authFile = conf.get("http.auth.file", "");
configureClient();
try {
setCredentials();
} catch (Exception ex) {
if (LOG.isFatalEnabled()) {
* @param realm
.setCredentials(authScope, credentials);
LOG.fatal("Could not read " + authFile + " : "
+ ex.getMessage());
ex.printStackTrace(LogUtil.getErrorStream(LOG));
}
}
}
/**
* Main method.
*
* @param args
* Command line arguments
*/
public static void main(String[] args) throws Exception {
Http http = new Http();
http.setConf(NutchConfiguration.create());
main(http, args);
}
/**
* Fetches the <code>url</code> with a configured HTTP client and gets the
* response.
*
* @param url
* URL to be fetched
* @param datum
* Crawl data
* @param redirect
* Follow redirects if and only if true
* @return HTTP response
*/
protected Response getResponse(URL url, WebPage page, boolean redirect)
throws ProtocolException, IOException {
resolveCredentials(url);
return new HttpResponse(this, url, page, redirect);
}
/**
* Configures the HTTP client
*/
private void configureClient() {
// Set up an HTTPS socket factory that accepts self-signed certs.
Protocol https = new Protocol("https",
new DummySSLProtocolSocketFactory(), 443);
Protocol.registerProtocol("https", https);
HttpConnectionManagerParams params = connectionManager.getParams();
params.setConnectionTimeout(timeout);
params.setSoTimeout(timeout);
params.setSendBufferSize(BUFFER_SIZE);
params.setReceiveBufferSize(BUFFER_SIZE);
params.setMaxTotalConnections(maxThreadsTotal);
if (maxThreadsTotal > maxThreadsPerHost) {
params.setDefaultMaxConnectionsPerHost(maxThreadsPerHost);
} else {
params.setDefaultMaxConnectionsPerHost(maxThreadsTotal);
}
// executeMethod(HttpMethod) seems to ignore the connection timeout on
// the connection manager.
// set it explicitly on the HttpClient.
client.getParams().setConnectionManagerTimeout(timeout);
HostConfiguration hostConf = client.getHostConfiguration();
ArrayList headers = new ArrayList();
// Set the User Agent in the header
headers.add(new Header("User-Agent", userAgent));
// prefer English
headers.add(new Header("Accept-Language",
"en-us,en-gb,en;q=0.7,*;q=0.3"));
// prefer UTF-8
headers.add(new Header("Accept-Charset",
"utf-8,ISO-8859-1;q=0.7,*;q=0.7"));
// prefer understandable formats
headers.add(new Header(
"Accept",
"text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
// accept gzipped content
headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate"));
hostConf.getParams().setParameter("http.default-headers", headers);
// HTTP proxy server details
if (useProxy) {
hostConf.setProxy(proxyHost, proxyPort);
if (proxyUsername.length() > 0) {
AuthScope proxyAuthScope = getAuthScope(this.proxyHost,
this.proxyPort, this.proxyRealm);
NTCredentials proxyCredentials = new NTCredentials(
this.proxyUsername, this.proxyPassword, this.agentHost,
this.proxyRealm);
client.getState().setProxyCredentials(proxyAuthScope,
proxyCredentials);
}
}
}
/**
* Reads authentication configuration file (defined as 'http.auth.file' in
* Nutch configuration file) and sets the credentials for the configured
* authentication scopes in the HTTP client object.
*
* @throws ParserConfigurationException
* If a document builder can not be created.
* @throws SAXException
* If any parsing error occurs.
* @throws IOException
* If any I/O error occurs.
*/
private static synchronized void setCredentials()
throws ParserConfigurationException, SAXException, IOException {
if (authRulesRead)
return;
authRulesRead = true; // Avoid re-attempting to read
InputStream is = conf.getConfResourceAsInputStream(authFile);
if (is != null) {
Document doc = DocumentBuilderFactory.newInstance()
.newDocumentBuilder().parse(is);
Element rootElement = doc.getDocumentElement();
if (!"auth-configuration".equals(rootElement.getTagName())) {
if (LOG.isWarnEnabled())
LOG.warn("Bad auth conf file: root element <"
+ rootElement.getTagName() + "> found in "
+ authFile + " - must be <auth-configuration>");
}
// For each set of credentials
NodeList credList = rootElement.getChildNodes();
for (int i = 0; i < credList.getLength(); i++) {
Node credNode = credList.item(i);
if (!(credNode instanceof Element))
continue;
Element credElement = (Element) credNode;
if (!"credentials".equals(credElement.getTagName())) {
if (LOG.isWarnEnabled())
LOG.warn("Bad auth conf file: Element <"
+ credElement.getTagName()
+ "> not recognized in " + authFile
+ " - expected <credentials>");
continue;
}
String username = credElement.getAttribute("username");
String password = credElement.getAttribute("password");
// For each authentication scope
NodeList scopeList = credElement.getChildNodes();
for (int j = 0; j < scopeList.getLength(); j++) {
Node scopeNode = scopeList.item(j);
if (!(scopeNode instanceof Element))
continue;
Element scopeElement = (Element) scopeNode;
if ("default".equals(scopeElement.getTagName())) {
// Determine realm and scheme, if any
String realm = scopeElement.getAttribute("realm");
String scheme = scopeElement.getAttribute("scheme");
// Set default credentials
defaultUsername = username;
defaultPassword = password;
defaultRealm = realm;
defaultScheme = scheme;
if (LOG.isTraceEnabled()) {
LOG.trace("Credentials - username: " + username
+ "; set as default" + " for realm: "
+ realm + "; scheme: " + scheme);
}
} else if ("authscope".equals(scopeElement.getTagName())) {
// Determine authentication scope details
String host = scopeElement.getAttribute("host");
int port = -1; // For setting port to AuthScope.ANY_PORT
try {
port = Integer.parseInt(scopeElement
.getAttribute("port"));
} catch (Exception ex) {
// do nothing, port is already set to any port
}
String realm = scopeElement.getAttribute("realm");
String scheme = scopeElement.getAttribute("scheme");
// Set credentials for the determined scope
AuthScope authScope = getAuthScope(host, port, realm,
scheme);
NTCredentials credentials = new NTCredentials(username,
password, agentHost, realm);
client.getState()
* Port number.
if (LOG.isTraceEnabled()) {
LOG.trace("Credentials - username: " + username
+ "; set for AuthScope - " + "host: "
+ host + "; port: " + port + "; realm: "
+ realm + "; scheme: " + scheme);
}
} else {
if (LOG.isWarnEnabled())
LOG.warn("Bad auth conf file: Element <"
+ scopeElement.getTagName()
+ "> not recognized in " + authFile
+ " - expected <authscope>");
}
}
is.close();
}
}
}
/**
* If credentials for the authentication scope determined from the specified
* <code>url</code> is not already set in the HTTP client, then this method
* sets the default credentials to fetch the specified <code>url</code>. If
* credentials are found for the authentication scope, the method returns
* without altering the client.
*
* @param url
* URL to be fetched
*/
private void resolveCredentials(URL url) {
if (defaultUsername != null && defaultUsername.length() > 0) {
int port = url.getPort();
if (port == -1) {
if ("https".equals(url.getProtocol()))
port = 443;
else
port = 80;
}
AuthScope scope = new AuthScope(url.getHost(), port);
if (client.getState().getCredentials(scope) != null) {
if (LOG.isTraceEnabled())
LOG.trace("Pre-configured credentials with scope - host: "
+ url.getHost() + "; port: " + port
+ "; found for url: " + url);
// Credentials are already configured, so do nothing and return
return;
}
if (LOG.isTraceEnabled())
LOG.trace("Pre-configured credentials with scope - host: "
+ url.getHost() + "; port: " + port
+ "; not found for url: " + url);
AuthScope serverAuthScope = getAuthScope(url.getHost(), port,
defaultRealm, defaultScheme);
NTCredentials serverCredentials = new NTCredentials(
defaultUsername, defaultPassword, agentHost, defaultRealm);
client.getState()
.setCredentials(serverAuthScope, serverCredentials);
}
}
/**
* Returns an authentication scope for the specified <code>host</code>,
* <code>port</code>, <code>realm</code> and <code>scheme</code>.
*
* @param host
* Host name or address.
* Authentication realm.
* @param scheme
* Authentication scheme.
*/
private static AuthScope getAuthScope(String host, int port, String realm,
String scheme) {
if (host.length() == 0)
host = null;
if (port < 0)
port = -1;
if (realm.length() == 0)
realm = null;
if (scheme.length() == 0)
scheme = null;
return new AuthScope(host, port, realm, scheme);
}
/**
* Returns an authentication scope for the specified <code>host</code>,
* <code>port</code> and <code>realm</code>.
*
* @param host
* Host name or address.
* @param port
* Port number.
* Authentication realm.
*/
private static AuthScope getAuthScope(String host, int port, String realm) {
return getAuthScope(host, port, realm, "");
}
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> */
public class Http extends HttpBase {
public static final Log LOG = LogFactory.getLog(Http.class);
private static MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
// Since the Configuration has not yet been set,
// then an unconfigured client is returned.
private static HttpClient client = new HttpClient(connectionManager);
private static String defaultUsername;
private static String defaultPassword;
private static String defaultRealm;
+ " - expected <credentials>");
continue;
private static String defaultScheme;
private static String authFile;
private static String agentHost;
private static boolean authRulesRead = false;
private static Configuration conf;
int maxThreadsTotal = 10;
private String proxyUsername;
private String proxyPassword;
private String proxyRealm;
private static final Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
static {
FIELDS.add(WebPage.Field.MODIFIED_TIME);
FIELDS.add(WebPage.Field.HEADERS);
}
@Override
public Collection<Field> getFields() {
return FIELDS;
}
/**
* Returns the configured HTTP client.
*
* @return HTTP client
*/
static synchronized HttpClient getClient() {
return client;
}
/**
* Constructs this plugin.
*/
public Http() {
super(LOG);
}
/**
* Reads the configuration from the Nutch configuration files and sets the
* configuration.
*
* @param conf
* Configuration
*/
public void setConf(Configuration conf) {
super.setConf(conf);
this.conf = conf;
this.maxThreadsTotal = conf.getInt("fetcher.threads.fetch", 10);
this.proxyUsername = conf.get("http.proxy.username", "");
this.proxyPassword = conf.get("http.proxy.password", "");
this.proxyRealm = conf.get("http.proxy.realm", "");
agentHost = conf.get("http.agent.host", "");
authFile = conf.get("http.auth.file", "");
configureClient();
try {
setCredentials();
} catch (Exception ex) {
if (LOG.isFatalEnabled()) {
LOG.fatal("Could not read " + authFile + " : "
+ ex.getMessage());
ex.printStackTrace(LogUtil.getErrorStream(LOG));
}
}
}
/**
* Main method.
*
* @param args
* Command line arguments
*/
public static void main(String[] args) throws Exception {
Http http = new Http();
http.setConf(NutchConfiguration.create());
main(http, args);
}
/**
* Fetches the <code>url</code> with a configured HTTP client and gets the
* response.
*
* @param url
* URL to be fetched
* @param datum
* Crawl data
* @param redirect
* Follow redirects if and only if true
* @return HTTP response
*/
protected Response getResponse(URL url, WebPage page, boolean redirect)
throws ProtocolException, IOException {
resolveCredentials(url);
return new HttpResponse(this, url, page, redirect);
}
/**
* Configures the HTTP client
*/
private void configureClient() {
// Set up an HTTPS socket factory that accepts self-signed certs.
Protocol https = new Protocol("https",
new DummySSLProtocolSocketFactory(), 443);
Protocol.registerProtocol("https", https);
HttpConnectionManagerParams params = connectionManager.getParams();
params.setConnectionTimeout(timeout);
params.setSoTimeout(timeout);
}
params.setSendBufferSize(BUFFER_SIZE);
params.setReceiveBufferSize(BUFFER_SIZE);
params.setMaxTotalConnections(maxThreadsTotal);
if (maxThreadsTotal > maxThreadsPerHost) {
params.setDefaultMaxConnectionsPerHost(maxThreadsPerHost);
} else {
params.setDefaultMaxConnectionsPerHost(maxThreadsTotal);
}
// executeMethod(HttpMethod) seems to ignore the connection timeout on
// the connection manager.
// set it explicitly on the HttpClient.
client.getParams().setConnectionManagerTimeout(timeout);
HostConfiguration hostConf = client.getHostConfiguration();
ArrayList headers = new ArrayList();
// Set the User Agent in the header
headers.add(new Header("User-Agent", userAgent));
// prefer English
headers.add(new Header("Accept-Language",
"en-us,en-gb,en;q=0.7,*;q=0.3"));
// prefer UTF-8
headers.add(new Header("Accept-Charset",
"utf-8,ISO-8859-1;q=0.7,*;q=0.7"));
// prefer understandable formats
headers.add(new Header(
"Accept",
"text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
// accept gzipped content
headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate"));
hostConf.getParams().setParameter("http.default-headers", headers);
// HTTP proxy server details
if (useProxy) {
hostConf.setProxy(proxyHost, proxyPort);
if (proxyUsername.length() > 0) {
AuthScope proxyAuthScope = getAuthScope(this.proxyHost,
this.proxyPort, this.proxyRealm);
NTCredentials proxyCredentials = new NTCredentials(
this.proxyUsername, this.proxyPassword, this.agentHost,
this.proxyRealm);
client.getState().setProxyCredentials(proxyAuthScope,
proxyCredentials);
}
}
}
/**
* Reads authentication configuration file (defined as 'http.auth.file' in
* Nutch configuration file) and sets the credentials for the configured
* authentication scopes in the HTTP client object.
*
* @throws ParserConfigurationException
* If a document builder can not be created.
* @throws SAXException
* If any parsing error occurs.
* @throws IOException
* If any I/O error occurs.
*/
private static synchronized void setCredentials()
throws ParserConfigurationException, SAXException, IOException {
if (authRulesRead)
return;
authRulesRead = true; // Avoid re-attempting to read
InputStream is = conf.getConfResourceAsInputStream(authFile);
if (is != null) {
Document doc = DocumentBuilderFactory.newInstance()
.newDocumentBuilder().parse(is);
Element rootElement = doc.getDocumentElement();
if (!"auth-configuration".equals(rootElement.getTagName())) {
if (LOG.isWarnEnabled())
LOG.warn("Bad auth conf file: root element <"
+ rootElement.getTagName() + "> found in "
+ authFile + " - must be <auth-configuration>");
}
// For each set of credentials
NodeList credList = rootElement.getChildNodes();
for (int i = 0; i < credList.getLength(); i++) {
Node credNode = credList.item(i);
if (!(credNode instanceof Element))
continue;
Element credElement = (Element) credNode;
if (!"credentials".equals(credElement.getTagName())) {
if (LOG.isWarnEnabled())
LOG.warn("Bad auth conf file: Element <"
+ credElement.getTagName()
+ "> not recognized in " + authFile
String username = credElement.getAttribute("username");
String password = credElement.getAttribute("password");
// For each authentication scope
NodeList scopeList = credElement.getChildNodes();
for (int j = 0; j < scopeList.getLength(); j++) {
Node scopeNode = scopeList.item(j);
if (!(scopeNode instanceof Element))
continue;
Element scopeElement = (Element) scopeNode;
if ("default".equals(scopeElement.getTagName())) {
// Determine realm and scheme, if any
String realm = scopeElement.getAttribute("realm");
String scheme = scopeElement.getAttribute("scheme");
// Set default credentials
defaultUsername = username;
defaultPassword = password;
defaultRealm = realm;
defaultScheme = scheme;
if (LOG.isTraceEnabled()) {
LOG.trace("Credentials - username: " + username
+ "; set as default" + " for realm: "
+ realm + "; scheme: " + scheme);
}
} else if ("authscope".equals(scopeElement.getTagName())) {
// Determine authentication scope details
String host = scopeElement.getAttribute("host");
int port = -1; // For setting port to AuthScope.ANY_PORT
try {
port = Integer.parseInt(scopeElement
.getAttribute("port"));
} catch (Exception ex) {
// do nothing, port is already set to any port
}
String realm = scopeElement.getAttribute("realm");
String scheme = scopeElement.getAttribute("scheme");
// Set credentials for the determined scope
AuthScope authScope = getAuthScope(host, port, realm,
scheme);
NTCredentials credentials = new NTCredentials(username,
password, agentHost, realm);
client.getState()
.setCredentials(authScope, credentials);
if (LOG.isTraceEnabled()) {
LOG.trace("Credentials - username: " + username
+ "; set for AuthScope - " + "host: "
+ host + "; port: " + port + "; realm: "
+ realm + "; scheme: " + scheme);
}
} else {
if (LOG.isWarnEnabled())
LOG.warn("Bad auth conf file: Element <"
+ scopeElement.getTagName()
+ "> not recognized in " + authFile
+ " - expected <authscope>");
}
}
is.close();
}
}
}
/**
* If credentials for the authentication scope determined from the specified
* <code>url</code> is not already set in the HTTP client, then this method
* sets the default credentials to fetch the specified <code>url</code>. If
* credentials are found for the authentication scope, the method returns
* without altering the client.
*
* @param url
* URL to be fetched
*/
private void resolveCredentials(URL url) {
if (defaultUsername != null && defaultUsername.length() > 0) {
int port = url.getPort();
if (port == -1) {
if ("https".equals(url.getProtocol()))
port = 443;
else
port = 80;
}
AuthScope scope = new AuthScope(url.getHost(), port);
if (client.getState().getCredentials(scope) != null) {
if (LOG.isTraceEnabled())
LOG.trace("Pre-configured credentials with scope - host: "
+ url.getHost() + "; port: " + port
+ "; found for url: " + url);
// Credentials are already configured, so do nothing and return
return;
}
if (LOG.isTraceEnabled())
LOG.trace("Pre-configured credentials with scope - host: "
+ url.getHost() + "; port: " + port
+ "; not found for url: " + url);
AuthScope serverAuthScope = getAuthScope(url.getHost(), port,
defaultRealm, defaultScheme);
NTCredentials serverCredentials = new NTCredentials(
defaultUsername, defaultPassword, agentHost, defaultRealm);
client.getState()
.setCredentials(serverAuthScope, serverCredentials);
}
}
/**
* Returns an authentication scope for the specified <code>host</code>,
* <code>port</code>, <code>realm</code> and <code>scheme</code>.
*
* @param host
* Host name or address.
* @param port
* Port number.
* @param realm
* Authentication realm.
* @param scheme
* Authentication scheme.
*/
private static AuthScope getAuthScope(String host, int port, String realm,
String scheme) {
if (host.length() == 0)
host = null;
if (port < 0)
port = -1;
if (realm.length() == 0)
realm = null;
if (scheme.length() == 0)
scheme = null;
return new AuthScope(host, port, realm, scheme);
}
/**
* Returns an authentication scope for the specified <code>host</code>,
* <code>port</code> and <code>realm</code>.
*
* @param host
* Host name or address.
* @param port
* Port number.
* @param realm
* Authentication realm.
*/
private static AuthScope getAuthScope(String host, int port, String realm) {
return getAuthScope(host, port, realm, "");
}
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">Http.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Annotation</td>
</tr>
<tr>
<td data-title="KindConflict"> Attribute</td>
</tr>
<tr>
<td data-title="KindConflict"> Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> Method declaration</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Static initializer</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>import java.net.URL;
import junit.framework.TestCase;
<<<<<<< HEAD
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.ContextHandler;
import org.mortbay.jetty.handler.ResourceHandler;
import org.mortbay.jetty.servlet.ServletHandler;
import org.mortbay.jetty.servlet.SessionHandler;
=======
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.storage.WebPage;</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>import java.net.URL;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.storage.WebPage;</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestProtocolHttpClient.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Import</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> startServer(47500);
*/
*/
public class TestProtocolHttpClient extends TestCase {
<<<<<<< HEAD
private Server server;
private Configuration conf;
private static final String RES_DIR = System.getProperty("test.data", ".");
private int port;
private Http http = new Http();
protected void setUp() throws Exception {
ContextHandler context = new ContextHandler();
context.setContextPath("/");
context.setResourceBase(RES_DIR);
ServletHandler sh = new ServletHandler();
sh.addServlet("org.apache.jasper.servlet.JspServlet", "*.jsp");
context.addHandler(sh);
context.addHandler(new SessionHandler());
server = new Server();
server.addHandler(context);
conf = new Configuration();
conf.addResource("nutch-default.xml");
conf.addResource("nutch-site-test.xml");
http = new Http();
http.setConf(conf);
}
protected void tearDown() throws Exception {
server.stop();
}
/**
* Tests whether the client can remember cookies.
*
* @throws Exception If an error occurs or the test case fails.
*/
public void testCookies() throws Exception {
startServer(47500);
fetchPage("/cookies.jsp", 200);
fetchPage("/cookies.jsp?cookie=yes", 200);
}
/**
* Tests that no pre-emptive authorization headers are sent by the
* client.
*
* @throws Exception If an error occurs or the test case fails.
*/
public void testNoPreemptiveAuth() throws Exception {
fetchPage("/noauth.jsp", 200);
}
/**
* Tests default credentials.
*
* @throws Exception If an error occurs or the test case fails.
*/
public void testDefaultCredentials() throws Exception {
startServer(47502);
fetchPage("/basic.jsp", 200);
}
/**
* Tests basic authentication scheme for various realms.
*
* @throws Exception If an error occurs or the test case fails.
*/
public void testBasicAuth() throws Exception {
startServer(47500);
fetchPage("/basic.jsp", 200);
fetchPage("/basic.jsp?case=1", 200);
fetchPage("/basic.jsp?case=2", 200);
server.start();
}
/**
* Tests that authentication happens for a defined realm and not for
public void testDigestAuth() throws Exception {
startServer(47500);
* other realms for a host:port when an extra <code>authscope</code>
* tag is not defined to match all other realms.
*
* @throws Exception If an error occurs or the test case fails.
*/
public void testOtherRealmsNoAuth() throws Exception {
startServer(47501);
fetchPage("/basic.jsp", 200);
fetchPage("/basic.jsp?case=1", 401);
fetchPage("/basic.jsp?case=2", 401);
}
/**
* Tests Digest authentication scheme.
*
* @throws Exception If an error occurs or the test case fails.
*/
public void testDigestAuth() throws Exception {
startServer(47500);
fetchPage("/digest.jsp", 200);
}
/**
* Tests NTLM authentication scheme.
*
* @throws Exception If an error occurs or the test case fails.
*/
public void testNtlmAuth() throws Exception {
startServer(47501);
fetchPage("/ntlm.jsp", 200);
}
/**
* Starts the Jetty server at a specified port.
*
* @param portno Port number.
* @throws Exception When an error occurs.
*/
private void startServer(int portno) throws Exception {
port = portno;
SocketConnector listener = new SocketConnector();
listener.setHost("127.0.0.1");
listener.setPort(port);
server.addConnector(listener);
server.start();
}
/**
* Fetches the specified <code>page</code> from the local Jetty server
* and checks whether the HTTP response status code matches with the
* expected code.
*
* @param page Page to be fetched.
* @param expectedCode HTTP response status code expected while
* fetching the page.
* @throws Exception When an error occurs or test case fails.
*/
private void fetchPage(String page, int expectedCode)
throws Exception {
URL url = new URL("http", "127.0.0.1", port, page);
Response response = null;
response = http.getResponse(url, new CrawlDatum(), true);
int code = response.getCode();
assertEquals("HTTP Status Code for " + url, expectedCode, code);
}
/**
* Returns an URL to the specified page.
*
* @param page Page available in the local Jetty
* server.
* @throws MalformedURLException If an URL can not be formed.
*/
private URL getURL(String page) throws MalformedURLException {
return new URL("http", "127.0.0.1", port, page);
}
=======
private Server server;
private Configuration conf;
private static final String RES_DIR = System.getProperty("test.data", ".");
private int port;
private Http http = new Http();
protected void setUp() throws Exception {
server = new Server();
// Context scontext = new Context();
// scontext.setContextPath("/");
// scontext.setResourceBase(RES_DIR);
// // servlet handler?
// scontext.addServlet("JSP", "*.jsp",
// "org.apache.jasper.servlet.JspServlet");
// scontext.addHandler(new ResourceHandler());
Context root = new Context(server,"/",Context.SESSIONS);
root.setContextPath("/");
root.setResourceBase(RES_DIR);
ServletHolder sh = new ServletHolder(org.apache.jasper.servlet.JspServlet.class);
root.addServlet(sh, "*.jsp");
conf = new Configuration();
conf.addResource("nutch-default.xml");
conf.addResource("nutch-site-test.xml");
http = new Http();
http.setConf(conf);
}
protected void tearDown() throws Exception {
server.stop();
}
/**
* Tests whether the client can remember cookies.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testCookies() throws Exception {
startServer(47500);
fetchPage("/cookies.jsp", 200);
fetchPage("/cookies.jsp?cookie=yes", 200);
}
/**
* Tests that no pre-emptive authorization headers are sent by the client.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testNoPreemptiveAuth() throws Exception {
startServer(47500);
fetchPage("/noauth.jsp", 200);
}
/**
* Tests default credentials.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testDefaultCredentials() throws Exception {
startServer(47502);
fetchPage("/basic.jsp", 200);
}
/**
* Tests basic authentication scheme for various realms.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testBasicAuth() throws Exception {
startServer(47500);
fetchPage("/basic.jsp", 200);
fetchPage("/basic.jsp?case=1", 200);
fetchPage("/basic.jsp?case=2", 200);
server.start();
}
/**
* Tests that authentication happens for a defined realm and not for other
* realms for a host:port when an extra <code>authscope</code> tag is not
* defined to match all other realms.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testOtherRealmsNoAuth() throws Exception {
startServer(47501);
fetchPage("/basic.jsp", 200);
fetchPage("/basic.jsp?case=1", 401);
fetchPage("/basic.jsp?case=2", 401);
}
/**
* Tests Digest authentication scheme.
*
* @throws Exception
* If an error occurs or the test case fails.
fetchPage("/digest.jsp", 200);
}
/**
* Tests NTLM authentication scheme.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testNtlmAuth() throws Exception {
startServer(47501);
fetchPage("/ntlm.jsp", 200);
}
/**
* Starts the Jetty server at a specified port.
*
* @param portno
* Port number.
* @throws Exception
* When an error occurs.
*/
private void startServer(int portno) throws Exception {
port = portno;
SelectChannelConnector connector1 = new SelectChannelConnector();
connector1.setHost("127.0.0.1");
connector1.setPort(port);
server.addConnector(connector1);
server.start();
}
/**
* Fetches the specified <code>page</code> from the local Jetty server and
* checks whether the HTTP response status code matches with the expected
* code.
*
* @param page
* Page to be fetched.
* @param expectedCode
* HTTP response status code expected while fetching the page.
* @throws Exception
* When an error occurs or test case fails.
*/
private void fetchPage(String page, int expectedCode) throws Exception {
URL url = new URL("http", "127.0.0.1", port, page);
Response response = null;
response = http.getResponse(url, new WebPage(), true);
int code = response.getCode();
assertEquals("HTTP Status Code for " + url, expectedCode, code);
}
/**
* Returns an URL to the specified page.
*
* @param page
* Page available in the local Jetty server.
* @throws MalformedURLException
* If an URL can not be formed.
*/
private URL getURL(String page) throws MalformedURLException {
return new URL("http", "127.0.0.1", port, page);
}
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> */
public class TestProtocolHttpClient extends TestCase {
private Server server;
private Configuration conf;
private static final String RES_DIR = System.getProperty("test.data", ".");
private int port;
private Http http = new Http();
protected void setUp() throws Exception {
server = new Server();
// Context scontext = new Context();
// scontext.setContextPath("/");
// scontext.setResourceBase(RES_DIR);
// // servlet handler?
// scontext.addServlet("JSP", "*.jsp",
// "org.apache.jasper.servlet.JspServlet");
// scontext.addHandler(new ResourceHandler());
Context root = new Context(server,"/",Context.SESSIONS);
root.setContextPath("/");
root.setResourceBase(RES_DIR);
ServletHolder sh = new ServletHolder(org.apache.jasper.servlet.JspServlet.class);
root.addServlet(sh, "*.jsp");
conf = new Configuration();
conf.addResource("nutch-default.xml");
conf.addResource("nutch-site-test.xml");
http = new Http();
http.setConf(conf);
}
}
protected void tearDown() throws Exception {
server.stop();
/**
* Tests whether the client can remember cookies.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testCookies() throws Exception {
startServer(47500);
fetchPage("/cookies.jsp", 200);
fetchPage("/cookies.jsp?cookie=yes", 200);
}
/**
* Tests that no pre-emptive authorization headers are sent by the client.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testNoPreemptiveAuth() throws Exception {
startServer(47500);
fetchPage("/noauth.jsp", 200);
}
/**
* Tests default credentials.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testDefaultCredentials() throws Exception {
startServer(47502);
fetchPage("/basic.jsp", 200);
}
/**
* Tests basic authentication scheme for various realms.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testBasicAuth() throws Exception {
startServer(47500);
fetchPage("/basic.jsp", 200);
fetchPage("/basic.jsp?case=1", 200);
fetchPage("/basic.jsp?case=2", 200);
server.start();
}
/**
* Tests that authentication happens for a defined realm and not for other
* realms for a host:port when an extra <code>authscope</code> tag is not
* defined to match all other realms.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testOtherRealmsNoAuth() throws Exception {
startServer(47501);
fetchPage("/basic.jsp", 200);
fetchPage("/basic.jsp?case=1", 401);
fetchPage("/basic.jsp?case=2", 401);
}
/**
* Tests Digest authentication scheme.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testDigestAuth() throws Exception {
startServer(47500);
fetchPage("/digest.jsp", 200);
}
/**
* Tests NTLM authentication scheme.
*
* @throws Exception
* If an error occurs or the test case fails.
*/
public void testNtlmAuth() throws Exception {
startServer(47501);
fetchPage("/ntlm.jsp", 200);
}
/**
* Starts the Jetty server at a specified port.
*
* @param portno
* Port number.
* @throws Exception
* When an error occurs.
*/
private void startServer(int portno) throws Exception {
port = portno;
SelectChannelConnector connector1 = new SelectChannelConnector();
connector1.setHost("127.0.0.1");
connector1.setPort(port);
server.addConnector(connector1);
server.start();
}
/**
* Fetches the specified <code>page</code> from the local Jetty server and
* checks whether the HTTP response status code matches with the expected
* code.
*
* @param page
* Page to be fetched.
* @param expectedCode
* HTTP response status code expected while fetching the page.
* @throws Exception
* When an error occurs or test case fails.
*/
private void fetchPage(String page, int expectedCode) throws Exception {
URL url = new URL("http", "127.0.0.1", port, page);
Response response = null;
response = http.getResponse(url, new WebPage(), true);
int code = response.getCode();
assertEquals("HTTP Status Code for " + url, expectedCode, code);
}
/**
* Returns an URL to the specified page.
*
* @param page
* Page available in the local Jetty server.
* @throws MalformedURLException
* If an URL can not be formed.
*/
private URL getURL(String page) throws MalformedURLException {
return new URL("http", "127.0.0.1", port, page);
}
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestProtocolHttpClient.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Attribute</td>
</tr>
<tr>
<td data-title="KindConflict"> Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> Method declaration</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
<<<<<<< HEAD
import org.apache.nutch.collection.CollectionManager;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.Inlinks;
=======
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.NutchConfiguration;
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
public class SubcollectionIndexingFilter extends Configured implements
IndexingFilter {</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.NutchConfiguration;
public class SubcollectionIndexingFilter extends Configured implements
IndexingFilter {</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">SubcollectionIndexingFilter.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Import</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> doc.add(FIELD_NAME, collname);
}
<<<<<<< HEAD
public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException {
String sUrl = url.toString();
addSubCollectionField(doc, sUrl);
return doc;
}
=======
@Override
public Collection<Field> getFields() {
return new ArrayList<Field>();
}
@Override
public NutchDocument filter(NutchDocument doc, String url, WebPage page)
throws IndexingException {
addSubCollectionField(doc, url);
return doc;
}
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> doc.add(FIELD_NAME, collname);
}
@Override
public Collection<Field> getFields() {
return new ArrayList<Field>();
}
@Override
public NutchDocument filter(NutchDocument doc, String url, WebPage page)
throws IndexingException {
addSubCollectionField(doc, url);
return doc;
}
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">SubcollectionIndexingFilter.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Annotation</td>
</tr>
<tr>
<td data-title="KindConflict"> Method declaration</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
<<<<<<< HEAD
import org.apache.nutch.parse.Parse;
=======
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
import org.apache.nutch.util.URLUtil;
import org.apache.nutch.util.domain.DomainSuffix;
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.URLUtil;
import org.apache.nutch.util.domain.DomainSuffix;
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TLDIndexingFilter.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Import</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> public Configuration getConf() {
return this.conf;
}
<<<<<<< HEAD
=======
@Override
public Collection<Field> getFields() {
return fields;
}
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> public Configuration getConf() {
return this.conf;
}
@Override
public Collection<Field> getFields() {
return fields;
}
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TLDIndexingFilter.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Annotation</td>
</tr>
<tr>
<td data-title="KindConflict"> Method declaration</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.NutchDocument;
<<<<<<< HEAD
import org.apache.nutch.indexer.NutchField;
import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.protocol.Content;
=======
import org.apache.nutch.scoring.ScoreDatum;
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
import org.apache.nutch.scoring.ScoringFilter;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.storage.WebPage;</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.scoring.ScoreDatum;
import org.apache.nutch.scoring.ScoringFilter;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.storage.WebPage;</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TLDScoringFilter.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Import</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>
@Override
*/
public class TLDScoringFilter implements ScoringFilter {
<<<<<<< HEAD
private Configuration conf;
private DomainSuffixes tldEntries;
public TLDScoringFilter() {
tldEntries = DomainSuffixes.getInstance();
}
public float indexerScore(Text url, NutchDocument doc, CrawlDatum dbDatum,
CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore)
throws ScoringFilterException {
NutchField tlds = doc.getField("tld");
float boost = 1.0f;
if(tlds != null) {
for(Object tld : tlds.getValues()) {
DomainSuffix entry = tldEntries.get(tld.toString());
if(entry != null)
boost *= entry.getBoost();
}
}
return initScore * boost;
}
public CrawlDatum distributeScoreToOutlink(Text fromUrl, Text toUrl,
ParseData parseData, CrawlDatum target, CrawlDatum adjust, int allCount,
int validCount) throws ScoringFilterException {
return adjust;
}
public float generatorSortValue(Text url, CrawlDatum datum, float initSort)
throws ScoringFilterException {
return initSort;
}
public void initialScore(Text url, CrawlDatum datum)
throws ScoringFilterException {
}
public void injectedScore(Text url, CrawlDatum datum)
throws ScoringFilterException {
}
public void passScoreAfterParsing(Text url, Content content, Parse parse)
throws ScoringFilterException {
}
public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content content)
throws ScoringFilterException {
}
public void updateDbScore(Text url, CrawlDatum old,
CrawlDatum datum, List<CrawlDatum> inlinked)
throws ScoringFilterException {
}
public Configuration getConf() {
return conf;
}
public void setConf(Configuration conf) {
this.conf = conf;
}
public CrawlDatum distributeScoreToOutlinks(Text fromUrl, ParseData parseData,
Collection<Entry<Text, CrawlDatum>> targets, CrawlDatum adjust,
int allCount) throws ScoringFilterException {
return adjust;
}
=======
private Configuration conf;
private DomainSuffixes tldEntries;
private final static Set<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
public TLDScoringFilter() {
tldEntries = DomainSuffixes.getInstance();
}
public Configuration getConf() {
return conf;
}
public void setConf(Configuration conf) {
this.conf = conf;
}
@Override
public Collection<WebPage.Field> getFields() {
return FIELDS;
}
@Override
public void injectedScore(String url, WebPage page)
throws ScoringFilterException {
}
public void initialScore(String url, WebPage page)
throws ScoringFilterException {
}
@Override
public float generatorSortValue(String url, WebPage page, float initSort)
throws ScoringFilterException {
return initSort;
}
@Override
public void distributeScoreToOutlinks(String fromUrl, WebPage page,
Collection<ScoreDatum> scoreData, int allCount)
throws ScoringFilterException {
}
@Override
public void updateScore(String url, WebPage page,
List<ScoreDatum> inlinkedScoreData) throws ScoringFilterException {
}
@Override
public float indexerScore(String url, NutchDocument doc, WebPage page,
float initScore) throws ScoringFilterException {
List<String> tlds = doc.getFieldValues("tld");
float boost = 1.0f;
if (tlds != null) {
for (String tld : tlds) {
DomainSuffix entry = tldEntries.get(tld);
if (entry != null)
boost *= entry.getBoost();
}
}
return initScore * boost;
}
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> */
public class TLDScoringFilter implements ScoringFilter {
private Configuration conf;
private DomainSuffixes tldEntries;
private final static Set<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();
public TLDScoringFilter() {
tldEntries = DomainSuffixes.getInstance();
}
public Configuration getConf() {
return conf;
}
public void setConf(Configuration conf) {
this.conf = conf;
}
@Override
public Collection<WebPage.Field> getFields() {
return FIELDS;
}
@Override
public void injectedScore(String url, WebPage page)
throws ScoringFilterException {
}
@Override
public void initialScore(String url, WebPage page)
throws ScoringFilterException {
}
@Override
public float generatorSortValue(String url, WebPage page, float initSort)
throws ScoringFilterException {
return initSort;
}
@Override
public void distributeScoreToOutlinks(String fromUrl, WebPage page,
Collection<ScoreDatum> scoreData, int allCount)
throws ScoringFilterException {
}
@Override
public void updateScore(String url, WebPage page,
List<ScoreDatum> inlinkedScoreData) throws ScoringFilterException {
}
@Override
public float indexerScore(String url, NutchDocument doc, WebPage page,
float initScore) throws ScoringFilterException {
List<String> tlds = doc.getFieldValues("tld");
float boost = 1.0f;
if (tlds != null) {
for (String tld : tlds) {
DomainSuffix entry = tldEntries.get(tld);
if (entry != null)
boost *= entry.getBoost();
}
}
return initScore * boost;
}
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TLDScoringFilter.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Annotation</td>
</tr>
<tr>
<td data-title="KindConflict"> Attribute</td>
</tr>
<tr>
<td data-title="KindConflict"> Method declaration</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> * limitations under the License.
*/
<<<<<<< HEAD
// $Id$
=======
// $Id: PrefixURLFilter.java 823614 2009-10-09 17:02:32Z ab $
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
package org.apache.nutch.urlfilter.prefix;
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> * limitations under the License.
*/
// $Id: PrefixURLFilter.java 823614 2009-10-09 17:02:32Z ab $
package org.apache.nutch.urlfilter.prefix;
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">PrefixURLFilter.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> Package declaration</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> webPageStore.flush();
Configuration myConfiguration = new Configuration(conf);
<<<<<<< HEAD
myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT, 1);
Path generatedSegment = generateFetchlist(Integer.MAX_VALUE,
myConfiguration, false);
=======
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 1);
myConfiguration.set(GeneratorJob.GENERATOR_COUNT_MODE, GeneratorJob.GENERATOR_COUNT_VALUE_HOST);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
ArrayList<URLWebPage> fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> webPageStore.flush();
Configuration myConfiguration = new Configuration(conf);
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 1);
myConfiguration.set(GeneratorJob.GENERATOR_COUNT_MODE, GeneratorJob.GENERATOR_COUNT_VALUE_HOST);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
ArrayList<URLWebPage> fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestGenerator.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> assertEquals(1, fetchList.size());
myConfiguration = new Configuration(conf);
<<<<<<< HEAD
myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT, 2);
generatedSegment = generateFetchlist(Integer.MAX_VALUE, myConfiguration,
false);
fetchlistPath = new Path(new Path(generatedSegment,
CrawlDatum.GENERATE_DIR_NAME), "part-00000");
=======
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 2);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> assertEquals(1, fetchList.size());
myConfiguration = new Configuration(conf);
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 2);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestGenerator.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> assertEquals(2, fetchList.size());
myConfiguration = new Configuration(conf);
<<<<<<< HEAD
myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT, 3);
generatedSegment = generateFetchlist(Integer.MAX_VALUE, myConfiguration,
false);
fetchlistPath = new Path(new Path(generatedSegment,
CrawlDatum.GENERATE_DIR_NAME), "part-00000");
=======
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 3);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> assertEquals(2, fetchList.size());
myConfiguration = new Configuration(conf);
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 3);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestGenerator.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>
/**
* Test that generator obeys the property "generator.max.count" and
<<<<<<< HEAD
* "generator.count.per.domain".
* @throws Exception
*/
public void testGenerateDomainLimit() throws Exception{
ArrayList<URLCrawlDatum> list = new ArrayList<URLCrawlDatum>();
list.add(createURLCrawlDatum("http://a.example.com/index.html", 1, 1));
list.add(createURLCrawlDatum("http://b.example.com/index.html", 1, 1));
list.add(createURLCrawlDatum("http://c.example.com/index.html", 1, 1));
createCrawlDB(list);
Configuration myConfiguration = new Configuration(conf);
myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT, 1);
myConfiguration.set(Generator.GENERATOR_COUNT_MODE, Generator.GENERATOR_COUNT_VALUE_DOMAIN);
Path generatedSegment = generateFetchlist(Integer.MAX_VALUE,
myConfiguration, false);
=======
* "generator.count.value=domain".
*
* @throws Exception
*/
public void testGenerateDomainLimit() throws Exception {
ArrayList<URLWebPage> list = new ArrayList<URLWebPage>();
list.add(createURLWebPage("http://one.example.com/index.html", 1, 1));
list.add(createURLWebPage("http://one.example.com/index1.html", 1, 1));
list.add(createURLWebPage("http://two.example.com/index.html", 1, 1));
list.add(createURLWebPage("http://two.example.com/index1.html", 1, 1));
list.add(createURLWebPage("http://three.example.com/index.html", 1, 1));
list.add(createURLWebPage("http://three.example.com/index1.html", 1, 1));
for (URLWebPage uwp : list) {
webPageStore.put(TableUtil.reverseUrl(uwp.getUrl()), uwp.getDatum());
}
webPageStore.flush();
Configuration myConfiguration = new Configuration(conf);
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 1);
myConfiguration.set(GeneratorJob.GENERATOR_COUNT_MODE, GeneratorJob.GENERATOR_COUNT_VALUE_DOMAIN);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>
/**
* Test that generator obeys the property "generator.max.count" and
* "generator.count.value=domain".
*
* @throws Exception
*/
public void testGenerateDomainLimit() throws Exception {
ArrayList<URLWebPage> list = new ArrayList<URLWebPage>();
list.add(createURLWebPage("http://one.example.com/index.html", 1, 1));
list.add(createURLWebPage("http://one.example.com/index1.html", 1, 1));
list.add(createURLWebPage("http://two.example.com/index.html", 1, 1));
list.add(createURLWebPage("http://two.example.com/index1.html", 1, 1));
list.add(createURLWebPage("http://three.example.com/index.html", 1, 1));
list.add(createURLWebPage("http://three.example.com/index1.html", 1, 1));
for (URLWebPage uwp : list) {
webPageStore.put(TableUtil.reverseUrl(uwp.getUrl()), uwp.getDatum());
}
webPageStore.flush();
Configuration myConfiguration = new Configuration(conf);
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 1);
myConfiguration.set(GeneratorJob.GENERATOR_COUNT_MODE, GeneratorJob.GENERATOR_COUNT_VALUE_DOMAIN);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestGenerator.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> For statement</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Method signature</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> assertEquals(1, fetchList.size());
myConfiguration = new Configuration(myConfiguration);
<<<<<<< HEAD
myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT, 2);
generatedSegment = generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
=======
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 2);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> assertEquals(1, fetchList.size());
myConfiguration = new Configuration(myConfiguration);
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 2);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestGenerator.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>
false);
assertEquals(2, fetchList.size());
myConfiguration = new Configuration(myConfiguration);
<<<<<<< HEAD
myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT, 3);
generatedSegment = generateFetchlist(Integer.MAX_VALUE, myConfiguration,
fetchlistPath = new Path(new Path(generatedSegment,
CrawlDatum.GENERATE_DIR_NAME), "part-00000");
=======
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 3);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> assertEquals(2, fetchList.size());
myConfiguration = new Configuration(myConfiguration);
myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 3);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
fetchList = CrawlTestUtil.readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestGenerator.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> private void generateFetchlist(int numResults, Configuration config,
boolean filter) throws Exception {
// generate segment
<<<<<<< HEAD
Generator g = new Generator(config);
Path[] generatedSegment = g.generate(dbDir, segmentsDir, -1, numResults,
Long.MAX_VALUE, filter, false);
if (generatedSegment==null) return null;
return generatedSegment[0];
=======
GeneratorJob g = new GeneratorJob();
g.setConf(config);
String crawlId = g.generate(numResults, System.currentTimeMillis(), filter, false);
if (crawlId == null)
throw new RuntimeException("Generator failed");
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}
/**</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> private void generateFetchlist(int numResults, Configuration config,
boolean filter) throws Exception {
// generate segment
GeneratorJob g = new GeneratorJob();
g.setConf(config);
String crawlId = g.generate(numResults, System.currentTimeMillis(), filter, false);
if (crawlId == null)
throw new RuntimeException("Generator failed");
}
/**</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestGenerator.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Array access</td>
</tr>
<tr>
<td data-title="KindConflict"> If statement</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Return statement</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> server.start();
}
<<<<<<< HEAD
protected void tearDown() throws Exception{
=======
public void tearDown() throws Exception{
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
server.stop();
fs.delete(testdir, true);
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> server.start();
}
public void tearDown() throws Exception{
server.stop();
fs.delete(testdir, true);
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestFetcher.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method signature</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> injector.inject(urlPath);
//generate
<<<<<<< HEAD
Generator g=new Generator(conf);
Path[] generatedSegment = g.generate(crawldbPath, segmentsPath, 1,
Long.MAX_VALUE, Long.MAX_VALUE, false, false);
=======
long time = System.currentTimeMillis();
GeneratorJob g = new GeneratorJob(conf);
String crawlId = g.generate(Long.MAX_VALUE, time, false, false);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
//fetch
<<<<<<< HEAD</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> injector.inject(urlPath);
//generate
long time = System.currentTimeMillis();
GeneratorJob g = new GeneratorJob(conf);
String crawlId = g.generate(Long.MAX_VALUE, time, false, false);
//fetch</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestFetcher.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
//fetch
<<<<<<< HEAD
Fetcher fetcher=new Fetcher(conf);
fetcher.fetch(generatedSegment[0], 1, true);
=======
time = System.currentTimeMillis();
conf.setBoolean(FetcherJob.PARSE_KEY, true);
FetcherJob fetcher = new FetcherJob(conf);
fetcher.fetch(1, crawlId, false, true);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
time = System.currentTimeMillis() - time;
</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>
//fetch
time = System.currentTimeMillis();
conf.setBoolean(FetcherJob.PARSE_KEY, true);
FetcherJob fetcher = new FetcherJob(conf);
fetcher.fetch(1, crawlId, false, true);
time = System.currentTimeMillis() - time;
</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestFetcher.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> conf.getFloat("fetcher.server.delay", 5));
assertTrue(time > minimumTime);
<<<<<<< HEAD
//verify content
Path content=new Path(new Path(generatedSegment[0], Content.DIR_NAME),"part-00000/data");
SequenceFile.Reader reader=new SequenceFile.Reader(fs, content, conf);
ArrayList<String> handledurls=new ArrayList<String>();
READ_CONTENT:
do {
Text key=new Text();
Content value=new Content();
if(!reader.next(key, value)) break READ_CONTENT;
String contentString=new String(value.getContent());
if(contentString.indexOf("Nutch fetcher test page")!=-1) {
handledurls.add(key.toString());
=======
List<URLWebPage> pages = CrawlTestUtil.readContents(webPageStore, Mark.FETCH_MARK, (String[])null);
assertEquals(urls.size(), pages.size());
List<String> handledurls = new ArrayList<String>();
for (URLWebPage up : pages) {
ByteBuffer bb = up.getDatum().getContent();
if (bb == null) {
continue;
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}
String content = new String(bb.array());
if (content.indexOf("Nutch fetcher test page")!=-1) {</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> conf.getFloat("fetcher.server.delay", 5));
assertTrue(time > minimumTime);
List<URLWebPage> pages = CrawlTestUtil.readContents(webPageStore, Mark.FETCH_MARK, (String[])null);
assertEquals(urls.size(), pages.size());
List<String> handledurls = new ArrayList<String>();
for (URLWebPage up : pages) {
ByteBuffer bb = up.getDatum().getContent();
if (bb == null) {
continue;
}
String content = new String(bb.array());
if (content.indexOf("Nutch fetcher test page")!=-1) {</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestFetcher.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> Continue statement</td>
</tr>
<tr>
<td data-title="KindConflict"> Do statement</td>
</tr>
<tr>
<td data-title="KindConflict"> For statement</td>
</tr>
<tr>
<td data-title="KindConflict"> If statement</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> Collections.sort(handledurls);
//verify that correct pages were handled
assertTrue(handledurls.containsAll(urls));
assertTrue(urls.containsAll(handledurls));
<<<<<<< HEAD
handledurls.clear();
//verify parse data
Path parseData = new Path(new Path(generatedSegment[0], ParseData.DIR_NAME),"part-00000/data");
reader = new SequenceFile.Reader(fs, parseData, conf);
READ_PARSE_DATA:
do {
Text key = new Text();
ParseData value = new ParseData();
if(!reader.next(key, value)) break READ_PARSE_DATA;
// make sure they all contain "nutch.segment.name" and "nutch.content.digest"
// keys in parse metadata
Metadata contentMeta = value.getContentMeta();
if (contentMeta.get(Nutch.SEGMENT_NAME_KEY) != null
&& contentMeta.get(Nutch.SIGNATURE_KEY) != null) {
handledurls.add(key.toString());
}
} while(true);
assertEquals(urls.size(), handledurls.size());
assertTrue(handledurls.containsAll(urls));
assertTrue(urls.containsAll(handledurls));
=======
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
}
private void addUrl(ArrayList<String> urls, String page) {</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> //verify that correct pages were handled
assertTrue(handledurls.containsAll(urls));
assertTrue(urls.containsAll(handledurls));
}
private void addUrl(ArrayList<String> urls, String page) {</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestFetcher.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Comment</td>
</tr>
<tr>
<td data-title="KindConflict"> Do statement</td>
</tr>
<tr>
<td data-title="KindConflict"> Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre>import org.apache.hadoop.mapred.JobConf;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
<<<<<<< HEAD
=======
import org.apache.nutch.util.NutchJobConf;
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
/**
* Unit tests for the plugin system</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre>import org.apache.hadoop.mapred.JobConf;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.NutchJobConf;
/**
* Unit tests for the plugin system</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestPluginSystem.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Import</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> public void testRepositoryCache() {
Configuration config = NutchConfiguration.create();
PluginRepository repo = PluginRepository.get(config);
<<<<<<< HEAD
JobConf job = new NutchJob(config);
=======
JobConf job = new NutchJobConf(config);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
PluginRepository repo1 = PluginRepository.get(job);
assertTrue(repo == repo1);
// now construct a config without UUID</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> public void testRepositoryCache() {
Configuration config = NutchConfiguration.create();
PluginRepository repo = PluginRepository.get(config);
JobConf job = new NutchJobConf(config);
PluginRepository repo1 = PluginRepository.get(job);
assertTrue(repo == repo1);
// now construct a config without UUID</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestPluginSystem.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
<div class="table-responsive-vertical shadow-z-1">
<table id="table" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Chunk</th>
</tr>
</thead>
</table>
<div>
<div class="table-responsive-vertical shadow-z-1" style="width: 80%; float: left; position: relative;">
<!-- Table starts here -->
<table id="tableConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Conflicting content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="ConflictingContent"><pre> config.addResource("nutch-default.xml");
config.addResource("nutch-site.xml");
repo = PluginRepository.get(config);
<<<<<<< HEAD
job = new NutchJob(config);
=======
job = new NutchJobConf(config);
>>>>>>> fc6a7f5eaeec70967b5845ee0079b660c3a0de8f
repo1 = PluginRepository.get(job);
assertTrue(repo1 != repo);
}</pre></td>
</tr>
</tbody>
</table>
<table id="tableSolution" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Solution content</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="SolutionContent"><pre> config.addResource("nutch-default.xml");
config.addResource("nutch-site.xml");
repo = PluginRepository.get(config);
job = new NutchJobConf(config);
repo1 = PluginRepository.get(job);
assertTrue(repo1 != repo);
}</pre></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="table-responsive-vertical shadow-z-1" style="width: 17.5%; float: right; position: relative;">
<table id="tableFileName" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>File</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="FileName">TestPluginSystem.java</td>
</tr>
</tbody>
</table>
<table id="tableDeveloperDecision" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Developer's decision</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="DeveloperDecision">Version 2</td>
</tr>
</tbody>
</table>
<table id="tableKindConflict" class="table table-hover table-mc-light-blue">
<thead>
<tr>
<th>Kind of conflict</th>
</tr>
</thead>
<tbody>
<tr>
<td data-title="KindConflict">Method invocation</td>
</tr>
<tr>
<td data-title="KindConflict"> Variable</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<!-- Última versão JavaScript compilada e minificada -->
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" ></script>
<script type="text/javascript" src="script.js"></script>
</body>
</html> |