Read two CSV files and remove the duplicate values within them.

Question

Hi,
I want to read two CSV files(which contains more than 100 rows and 100 columns) and remove the duplicate values within that two files and merge all the unique values and display it as a single file.
Can anyone help me out.
Thanks in advance.

Answer 1

kirthi wrote:
Can you help me....Yeah, I've just finished... Here's a skeleton of my solution.
The first thing I think you should do is write a line-parser which splits your input data up into fields, and test it.
Then fill out the below parse method, and test it with that debugPrint method.
Then go to work on the print method.
I can help a bit along the way, but if you want to do this then you have to do it yourself. I'm not going to do it for you.
Cheers. Keith.
package forums.kirthi;
import java.util.*;
import java.io.PrintStream;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import krc.utilz.io.ParseException;
import krc.utilz.io.Filez.LineParser;
import krc.utilz.io.Filez.CsvLineParser;
public class DistinctColumnValuesFromCsvFiles
  public static void main(String[] args) {
    if (args.length==0) args = new String[] {"input1.csv", "input2.csv"};
    try {
      // data is a Map of ColumnNames to Sets-Of-Values
      Map<String,Set<String>> data = new HashMap<String,Set<String>>();
      // add the contents of each file to the data
      for ( String filename : args ) {
        data.putAll(parse(filename));
      // print the data to output.csv
      print(data);
    } catch (Exception e) {
      e.printStackTrace();
  private static Map<String,Set<String>> parse(String filename) throws IOException, ParseException {
    BufferedReader reader = null;
    try {
      reader = new BufferedReader(new FileReader(filename));
      CsvLineParser.squeeze = true; // field.trim().replaceAll("\\s+"," ")
      LineParser<String[]> parser = new CsvLineParser();
      int lineNumber = 1;
      // 1. read the column names (first line of file) into a List
      // 2. read the column values (subsequent lines of file) into a List of Set's of String's
      // 3. build a Map of columnName --> columnValues and return it
    } finally {
      if(reader!=null)reader.close();
  private static void debugPrint(Map<String,Set<String>> data) {
    for ( Map.Entry<String,Set<String>> entry : data.entrySet() ) {
      System.out.println("DEBUG: "+entry.getKey()+" "+Arrays.toString(entry.getValue().toArray(new String[0])));
  private static void print(Map<String,Set<String>> data) {
    // 1. get the column names from the table.
    // 2. create a List of List's of String's called matrix; logically [COL][ROW]
    // 3. print the column names and add the List<String> for this col to the matrix
    // 4. print the matrix by inerating columns and then rows
}