public class SolrDeleteDuplicates extends Reducer<Text,SolrDeleteDuplicates.SolrRecord,Text,SolrDeleteDuplicates.SolrRecord> implements Tool
SolrDeleteDuplicates.SolrRecord
instances(which contain id, boost and timestamp)SolrDeleteDuplicates.SolrRecord
s with the same digest will be
grouped together. Now, of these documents with the same digests, delete all
of them except the one with the highest score (boost field). If two (or more)
documents have the same score, then the document with the latest timestamp is
kept. Again, every other is deleted from solr index.Modifier and Type | Class and Description |
---|---|
static class |
SolrDeleteDuplicates.SolrInputFormat |
static class |
SolrDeleteDuplicates.SolrInputSplit |
static class |
SolrDeleteDuplicates.SolrRecord |
static class |
SolrDeleteDuplicates.SolrRecordReader |
Reducer.Context
Modifier and Type | Field and Description |
---|---|
static org.slf4j.Logger |
LOG |
Constructor and Description |
---|
SolrDeleteDuplicates() |
Modifier and Type | Method and Description |
---|---|
void |
cleanup(Reducer.Context context) |
boolean |
dedup(String solrUrl) |
Configuration |
getConf() |
static void |
main(String[] args) |
void |
reduce(Text key,
Iterable<SolrDeleteDuplicates.SolrRecord> values,
Reducer.Context context) |
int |
run(String[] args) |
void |
setConf(Configuration conf) |
void |
setup(Reducer.Context job) |
public Configuration getConf()
getConf
in interface Configurable
public void setConf(Configuration conf)
setConf
in interface Configurable
public void setup(Reducer.Context job) throws IOException
setup
in class Reducer<Text,SolrDeleteDuplicates.SolrRecord,Text,SolrDeleteDuplicates.SolrRecord>
IOException
public void cleanup(Reducer.Context context) throws IOException
cleanup
in class Reducer<Text,SolrDeleteDuplicates.SolrRecord,Text,SolrDeleteDuplicates.SolrRecord>
IOException
public void reduce(Text key, Iterable<SolrDeleteDuplicates.SolrRecord> values, Reducer.Context context) throws IOException
reduce
in class Reducer<Text,SolrDeleteDuplicates.SolrRecord,Text,SolrDeleteDuplicates.SolrRecord>
IOException
public boolean dedup(String solrUrl) throws IOException, InterruptedException, ClassNotFoundException
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException
run
in interface Tool
IOException
InterruptedException
ClassNotFoundException
Copyright © 2015 The Apache Software Foundation