Unicode to String Conversion

Question

Dear all,
I'm trying to get the String from unicode, previously i'm converting the String to Unicode like below,
String s = "\\u"+Integer.toHexString(s.charAt(0));
Which is giving me the same what i appended above even after Conversion using UTF-8.
If i do String s ="\u30f3"
It is giving me correct character. But my problem is i'm getting the latter part i.e hexString latter in the run time...through Integer.toHexString..
i can't add this as a character.. so it is taking as a two different strings..
How to handle this situation..
Hope i can find the answer,
Thanks in advance,
Srinivas N

Answer 1

Below is my IOUtils class. In particular, see the loadTextFile method for an example of how to read a UTF file, and the isUTF method to detect if a file is Unicode
You will not be able to compile this class unless you have my InfoFetcher class. You should be able to find a reference to this class by searching google for "InfoFetcher tjacobs01"
You are welcome to use and modify this class, but please don't change the package or take credit for it as your own work
package tjacobs.io;
import java.awt.Component;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.swing.JOptionPane;
import javax.swing.JTable;
import javax.swing.table.TableModel;
import tjacobs.io.InfoFetcher.FetcherListener;
* IOUtils class
* This is an important class that I use a lot.<p>
* has a lot of static utilities. Most should be fairly self-explanatory
public class IOUtils {
     public static final int DEFAULT_BUFFER_SIZE = (int) Math.pow(2, 20); //1 MByte
     public static final int DEFAULT_WAIT_TIME = 30000;
     public static final boolean ALWAYS_BACKUP = false;
     public static String loadTextFile(File f) throws IOException {
          return loadTextFile(f, isUTF16(f));
     public static String loadTextFile(File f, boolean UTF16) throws IOException {
          BufferedReader br = !UTF16 ?
                    new BufferedReader(new FileReader(f)) :
                    new BufferedReader(
                         new InputStreamReader(new FileInputStream(f),
                                               "UTF-16"));
          int length = (int) f.length();
          char data[] = new char[!UTF16?(int)length : ((int)length) / 2 - 1];
          int got = 0;
          do {
               got += br.read(data, got, data.length - got);
          while (got < data.length);
          return new String(data);
     public static InfoFetcher loadData(InputStream in) {
          byte buf[] = new byte[DEFAULT_BUFFER_SIZE]; // 1 MByte
          return loadData(in, buf);
     public static InfoFetcher loadData(InputStream in, byte buf[]) {
          return loadData(in, buf, TimeOut.DEFAULT_WAIT_TIME);
     public static InfoFetcher loadData(InputStream in, byte buf[], int waitTime) {
          return new InfoFetcher(in, buf, waitTime);
     public static InfoFetcher loadData(InputStream in, int initBufLength, int waitTime) {
          return loadData(in, new byte[initBufLength], waitTime);
     public static InfoFetcher loadData(File f) throws FileNotFoundException{
          //if (!f.exists()) throw new FileNotFoundException(f.getAbsolutePath());
          //create the inputstream first so that we can generate the FileNotFoundException right away
          InputStream in = new BufferedInputStream(new FileInputStream(f));
          long len = f.length();
          if (len > Integer.MAX_VALUE) {
               throw new RuntimeException("File size exceeds maximum size for a byte buffer");
          return loadData(in, (int) len, TimeOut.NO_TIMEOUT);
     public static InfoFetcher loadData(URL url) throws IOException{
          return loadData(url.openConnection());
     public static InfoFetcher loadData(URLConnection conn) throws IOException {
          int size = conn.getContentLength();
          if (size < 0) return loadData(conn.getInputStream(), 2000, DEFAULT_WAIT_TIME);
          return loadData(conn.getInputStream(), size, DEFAULT_WAIT_TIME);
      * Note: There is no guarentee that this method will
      * ever return. For instance, if you call loadAll on
      * an open socket connection it won't return until the
      * socket has closed
     public static String loadAllString(InputStream in) {
          InfoFetcher fetcher = loadData(in);
          fetcher.run();
          return new String(fetcher.buf, 0, fetcher.got);
      * Note: There is no guarentee that this method will
      * ever return. For instance, if you call loadAll on
      * an open socket connection it won't return until the
      * socket has closed
     public static byte[] loadAll(InputStream in) {
          InfoFetcher fetcher = loadData(in);
          return fetcher.readCompletely();
     public static void copyBufs(byte src[], byte target[]) {
          int length = Math.min(src.length, target.length);
          for (int i = 0; i < length; i++) {
               target[i] = src;
     * Not threaded by default. If you need this to run
     * in a separate, create a new thread or runnable class
     * @param in
     * @param out
     public static void pipe (InputStream in, OutputStream out) {
          pipe (in, out, TimeOut.NO_TIMEOUT);
     * Not threaded by default. If you need this to run
     * in a separate, create a new thread or runnable class
     * @param in
     * @param out
     public static void pipe (InputStream in, final OutputStream out, int timeout) {
          pipe (in, out, timeout, false);
     private static class PipeFetcher implements FetcherListener {
          OutputStream out;
          InputStream in;
          public IOException ex;
          boolean closeWhenDone;
          public PipeFetcher (InputStream in, OutputStream out, boolean closeWhenDone) {
               this.out = out;
               this.closeWhenDone = closeWhenDone;
          public void fetchedMore(byte[] buf, int start, int end) {
               try {
                    out.write(buf, start, end - start);
               catch (IOException iox) {
                    ex = iox;
                    try {
                         in.close();
                         out.close();
                    catch (IOException iox2) {
                         iox2.printStackTrace();
          public void fetchedAll(byte[] buf) {
               if (closeWhenDone) {
                    try {
                         out.close();
                    catch (IOException iox) {
                         iox.printStackTrace();
     * Not threaded by default. If you need this to run
     * in a separate, create a new thread or runnable class
     * @param in
     * @param out
     public static IOException pipe (final InputStream in, final OutputStream out, int timeout, final boolean closeWhenDone) {
          InfoFetcher info = new InfoFetcher (in, new byte[DEFAULT_BUFFER_SIZE], timeout);
          PipeFetcher pf = new PipeFetcher(in, out, closeWhenDone);
          info.addFetcherListener(pf);
          info.run();
          return pf.ex;
          info.addInputStreamListener(new InputStreamListener() {
               public void gotMore(InputStreamEvent ev) {
                    try {
                         out.write(ev.getBytes(), ev.getStart(), ev.getBytesRetrieved() - ev.getStart());
                    catch (IOException iox) {
                         System.err.println ("Pipe closing");
               public void gotAll(InputStreamEvent ev) {}
          Thread t= new Thread (info);
          t.start();
     public static byte[] expandBuf(byte array[]) {
          return expandBuf(array, array.length * 2);
     public static byte[] expandBuf(byte array[], int newlength) {
          byte newbuf[] = new byte[newlength];
          copyBufs(array, newbuf);
          return newbuf;
     public static byte[] trimBuf(byte[] array, int size) {
          byte[] newbuf = new byte[size];
          for (int i = 0; i < size; i++) {
               newbuf[i] = array[i];
          return newbuf;
     * @see getFileOutputStream(File, boolean)
     public static OutputStream getFileOutputStream(File file) throws IOException {
          return getFileOutputStream(file, true);
     * Convienience method for opening a FileOutputStream w/wo a buffer
     * makes sure that the file directory exists so this should always succeed.
     public static OutputStream getFileOutputStream(File file, boolean buffered) throws IOException {
          if (!file.exists() && !file.isDirectory()) {
               confirmDirectoryExists(file.getParentFile());
          if (file.exists()) {
               if (ALWAYS_BACKUP) {
                    file.renameTo(new File(file.getAbsolutePath() + "~"));
               } else {
                    file.delete();
          file.createNewFile();
          OutputStream out = new FileOutputStream(file);
          if (buffered) {
               out = new BufferedOutputStream(out);
          return out;
     * Confirms that a directory exists and makes it if it doesn't
     public static void confirmDirectoryExists(File dir) {
          if (!dir.exists()) {
               confirmDirectoryExists(dir.getParentFile());
               dir.mkdir();
          if (!dir.isDirectory()) {
               confirmDirectoryExists(dir.getParentFile());
     public static OutputStream getFileOutputStream(String name) throws IOException {
          return getFileOutputStream(name, true);
     public static PrintStream getFilePrintStream(String file) throws IOException {
          return new PrintStream(getFileOutputStream(file));
     public static PrintStream getFilePrintStream(File file) throws IOException {
          return new PrintStream(getFileOutputStream(file));
     public static OutputStream getFileOutputStream(String name, boolean buffered) throws IOException {
          return getFileOutputStream(new File(name), buffered);
     * @param f if f is a directory it returns the absolue path to f otherwise it returns the absolute path to the directory f is in
     public static String getDirectory(File f) {
          if (f.isDirectory()) {
               return f.getAbsolutePath();
          else {
               return f.getParentFile().getAbsolutePath();
     * Get the file without the extension.
     * @see getFileNoExtension(String);
     public static String getFilenameNoExtension(File f) {
          return getFilenameNoExtension(f.getName());
     * Gets the file name without the extension
     * returns the whole file name if no '.' is found<br>
     * otherwise returns whatever's before the last .
     public static String getFilenameNoExtension(String s) {
          int idx = s.indexOf('.');
          if (idx == -1) {
               return s;
          else {
               return s.substring(0, idx);
     * gets the file extension
     * if a '.' character is found it returns what's after the last .
     * if not, it returns the empty string
     public static String getFileExtension(String s) {
          int idx = s.lastIndexOf('.');
          if (idx == -1) {
               return "";
          else {
               return s.substring(idx + 1);
     * @see getFileExtension(String)
     public static String getFileExtension(File f) {
          return getFileExtension(f.getName());
     * Delete everything in a directory. Recursively deletes all sub-directories
     public static void deleteDirectory (File f, Component parent) {
          if (!f.isDirectory()) {
               throw new RuntimeException("File " + f.getAbsolutePath() + " is not a directory!");
          int val = JOptionPane.showConfirmDialog(parent, "Confirm Delete " + f.getAbsolutePath(), "Confirm Delete " + f.getAbsolutePath(), JOptionPane.OK_CANCEL_OPTION);
          if (val == JOptionPane.OK_OPTION) {
               deleteAllFiles(f);
     private static void deleteAllFiles (File f) {
          //recursively delete all its contents
          if (!f.isDirectory()) {
               //throw new RuntimeException("File " + f.getAbsolutePath() + " is not a directory!");
               f.delete();
          else {
               File[] files = f.listFiles();
               for (int i = 0; i < files.length; i++) {
                    if (files[i].equals(f) || files[i].equals(f.getParent())) {
                         continue;
                    deleteAllFiles(files[i]);
               f.delete();
     * static utility method for copying a file to another location
     public static void copyFile (File src, File newParent) throws FileNotFoundException {
          if (!src.exists()) {
               return;
          if (!newParent.exists()) {
               newParent.mkdirs();
               //throw new RuntimeException("Parent folder must exist");
          if (newParent.isDirectory()) {
               File newFile = new File(newParent, src.getName());
               if (src.isDirectory()) {
                    newFile.mkdir();
                    File children[] = src.listFiles();
                    for (int i = 0; i < children.length; i++) {
                         copyFile(children[i], newFile);
               else {
                    //loadFile
                    InfoFetcher info = loadData(new FileInputStream(src));
                    final BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(newFile));
                    info.addInputStreamListener(new InputStreamListener() {
                         int lastbytes = 0;
                         public void gotMore(InputStreamEvent ev1) {
                              try {
                                   out.write(ev1.getBytes(), lastbytes, ev1.getBytesRetrieved() - lastbytes);
                                   lastbytes = ev1.getBytesRetrieved();
                              catch (IOException iox) {
                                   iox.printStackTrace();
                         public void gotAll(InputStreamEvent ev2) {
     * @deprecated use the Find class
     public static File find(Pattern p, File start) {
          return recursiveFind(start, null, p, true);
     * @deprecated use the Find class
     public static File find(Pattern p) {
          return find(p, new File("."));
     * @deprecated use the Find class
     private static File recursiveFind(File current, File cameFrom, Pattern p, boolean startDescending) {
          Matcher m = p.matcher(current.getName());
          if (m.matches()) {
               return current;
          File[] files = current.listFiles();
          if (startDescending) {
               File value = descend(current, cameFrom, p, startDescending);
               if (value != null) return value;
               return ascend(current, cameFrom, p, startDescending);
          else {
               File value = ascend(current, cameFrom, p, startDescending);
               if (value != null) return value;
               return descend(current, cameFrom, p, startDescending);               
     * @deprecated use the Find class
     private static File ascend(File current, File cameFrom, Pattern p, boolean startDescending) {
          File par = current.getParentFile();
          if (par == null) {
               return null;
          par = par.getAbsoluteFile();
          if (par.equals(cameFrom)) {
               return null;
          return recursiveFind(par, current, p, false);
     * @deprecated use the Find class
     private static File descend(File current, File cameFrom, Pattern p, boolean startDescending) {
          File files[] = current.listFiles();
          if (files == null) {
               return null;
          for (int i = 0; i < files.length; i++) {
               File child = files[i];
               if (child.equals(cameFrom)) {
                    continue;
               File f = recursiveFind(child, current, p, true);
               if (f != null) {
                    return f;
          return null;
     * Utility for saving a string to a file. Rather than have to set
     * up all the writers etc and trap exceptions, just returns a boolean
     * if it worked
     * <p>
     * @see saveData(File, byte[])
     public static boolean saveData(File f, String s) {
          return saveData(f, s.getBytes());
     * Utility for saving a string to a file. Rather than have to set
     * up all the writers etc and trap exceptions, just returns a boolean
     * if it worked
     public static boolean saveData(File f, byte[] data) {
          try {
               OutputStream out = new BufferedOutputStream(new FileOutputStream(f));
               out.write(data);
               out.close();
          catch(IOException iox) {
               iox.printStackTrace();
               return false;
          return true;
     * Way to save a stream to a file. Not multithreaded.
     * Will block until the save is done
     * @param f
     * @param in
     * @return
     public static boolean saveData(File f, InputStream in) throws FileNotFoundException {
          return pipe(in, new FileOutputStream(f), TimeOut.NO_TIMEOUT, true) == null;
          //InfoFetcher fetcher = loadData(in);
//          try {
//               FileOutputStream out = new FileOutputStream(f);
//               fetcher.addFetcherListener(new FetcherListener() {
//                    public void fetchedMore(byte[] buf, int start, int end) {
//                    public void fetchedAll(byte[] buf) {
//          } catch (IOException iox) {
//               iox.printStackTrace();
//               return false;
//          return true;
     * @deprecated use Find
     public static List<File> findAllFiles(Pattern p) {
          ArrayList<File> l = new ArrayList<File>();
          //File start = File.listRoots()[0];
          File start = new File("C:\\");
          return recursiveFindAll(start, null, p, true, l, 0);
          //l;
     * @deprecated use Find
     private static List recursiveFindAll(File current, File cameFrom, Pattern p, boolean startDescending, ArrayList<File> list, int level) {
          //System.out.println("" + level + " Scanning: " + current + "par: " + cameFrom);
          System.out.println("Scanning: " + current);
          Matcher m = p.matcher(current.getName());
          if (current.getName().equals("C:\\")) {
               System.out.println("root");
               try {
                    System.in.read();
               catch (IOException iox) {}
          if (m.matches()) {
               //return current;
               list.add(current);
               System.out.println("Adding " + current);
          File[] files = current.listFiles();
          if (startDescending) {
               //File value = descend(current, cameFrom, p, startDescending);
               descendAll(current, cameFrom, p, startDescending, list, level + 1);
               //ascendAll(current, cameFrom, p, startDescending, list, level + 1);
               //if (value != null) return value;
               //return ascend(current, cameFrom, p, startDescending);
          else {
               //ascendAll(current, cameFrom, p, startDescending, list, level + 1);
               descendAll(current, cameFrom, p, startDescending, list, level + 1);
               //File value = ascend(current, cameFrom, p, startDescending);
               //if (value != null) return value;
               //return descend(current, cameFrom, p, startDescending);               
          return list;
     * @deprecated use Find
     private static List ascendAll(File current, File cameFrom, Pattern p, boolean startDescending, ArrayList<File> list, int level) {
          File par = current.getParentFile();
          if (par == null) {
               return list;
          par = par.getAbsoluteFile();
          if (par.equals(cameFrom)) {
               return list;
          recursiveFindAll(par, current, p, false, list, level);
          return list;
     * @deprecated use Find
     private static File descendAll(File current, File cameFrom, Pattern p, boolean startDescending, ArrayList<File> list, int level) {
          File files[] = current.listFiles();
          if (files == null) {
               return null;
          for (int i = 0; i < files.length; i++) {
               File child = files[i];
               if (child.equals(cameFrom)) {
                    continue;
               recursiveFindAll(child, current, p, true, list, level);
          return null;
     public File getUniqueName(File f) {
          return getUniqueName (f, new MessageFormat("~{0,number,integer}"));
     public String getNameWOExtension(File f, boolean useAbsolute) {
          int idx = f.getName().indexOf(".");
          return (idx == -1) ? (useAbsolute ? f.getAbsolutePath() : f.getName()) : (useAbsolute ? f.getAbsolutePath() : f.getName()).substring(0, (useAbsolute ? f.getAbsolutePath().lastIndexOf(".") : f.getName().lastIndexOf(".")));
     public String getFileType (File f, boolean includeDot) {
          int idx = f.getName().lastIndexOf(".");
          return idx == -1 ? "" : f.getName().substring(idx + (includeDot ? 0 : 1));
     public File getUniqueName(File f, MessageFormat format) {
          String base = getNameWOExtension(f, true);
          String extension = getFileType(f, true);
          int count = 0;
          while (f.exists()) {
               count++;
               f = new File (base + format.format(new Object[] {Integer.valueOf(count)}) + extension);
          return f;
     public static boolean isUTF16 (File f) throws IOException {
          FileInputStream in = null;
          try {
               if (!f.exists() || f.length() < 2) return false;
               in = new FileInputStream(f);
               byte b = (byte)in.read();
               if (!(b == -1)) return false;
               b = (byte) in.read();
               return b == -2;
          finally {
               if (in != null) in.close();
     public static boolean isUTF16(PushbackInputStream in) {
          boolean got1 = false, got2 = false;
          byte b = 0, b2 = 0;
          try {
               b = (byte)in.read();
               got1 = true;
               if (!(b == -1 || b== -2)) {
                    return false;
               b2 = (byte) in.read();
               got2 = true;
               return b == -1 ? b2 == -2 : b2 == -1;
          catch (IOException iox) {
               iox.printStackTrace();
          finally {
               try {
                    if (got1) {
                         in.unread(b);
                    if (got2) {
                         in.unread(b2);
               } catch (IOException iox) {
                    iox.printStackTrace();
          return false;
     public void saveJTableToCSV(JTable table, File f) throws IOException {
          if (table == null || f == null) return;
          TableModel model = table.getModel();
          if (model == null) return;
          int rows = model.getRowCount();
          int columns = model.getColumnCount();
          PrintWriter pw = new PrintWriter(new FileWriter(f));
          for (int i = 0; i < rows; i++) {
               for (int j = 0; j < columns; j++) {
                    Object o = model.getValueAt(i, j);
                    pw.print(o.toString());
                    if (j != columns - 1) {
                         pw.print(",");
               pw.println();
     public static String getPathRelativeTo(File path, File relativeTo) {
          return getPathRelativeTo(path.getAbsolutePath(), relativeTo.getAbsolutePath());
     public static String getPathRelativeTo(String path, String relativeTo) {
          if (!path.startsWith(relativeTo)) return null;
          path = path.substring(relativeTo.length());
          if (path.startsWith(File.separator)) path = path.substring(File.separator.length());
          return path;