// Last edited on 2013-04-06 19:04:19 by stolfilocal package main; import java.awt.image.BufferedImage; import java.io.File; import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; import java.io.LineNumberReader; import java.io.ObjectInputStream; import java.util.ArrayList; import java.util.LinkedList; import javax.imageio.ImageIO; import parameters.detection_arguments; import parameters.io_arguments; import preprocessing.Parser; import preprocessing.Util; import regrouping.Groupment; import segmentation.Labels; import segmentation.Regions; import segmentation.Segmentation; import classification.Classify; import classification.Features; import classification.SVM; import classification.SVM_DATA; import fr.lip6.classifier.Classifier; import fr.lip6.classifier.DoublePegasosSVM; import fr.lip6.type.TrainingSample; public class DetectionThread extends Thread { /*Letters recognized by this thread, magnified for the scale {param.getScale()}*/ Regions[] regions; /*List of characters chains recognized by this thread in one image scale*/ LinkedList list; /*Image to be analyzed*/ BufferedImage input_image; /*Flag to control if the {input_image} is negated or not*/ boolean inversion; /*Flag that stores the image pyramid level that is being processed*/ int pyramid_level; /*Detection parameters used in the segmentation, classification and regrouping steps*/ detection_arguments parameters; ArrayList svm_descriptors; ArrayList svm_classifiers; /*Flag that controls the end of the thread*/ boolean finished = false; /**/ public Labels Labeling (int w, int h, int[] image) { Labels labels = new Labels (w, h); int number_of_labels = labels.create_labels_from_image (w, h, image); labels.setNumberOfRegions(number_of_labels); return labels; } void Write_Label_Image (Labels labels, String filename, int pyramid_level, detection_arguments detection_parameters) { BufferedImage image = labels.Get_Image_From_Labels (); if (detection_parameters.write_image) { try { ImageIO.write(image, "png", new File(Util.make_name (filename, inversion, pyramid_level, ".png"))); } catch (Exception e) { System.err.println("error: fail to save " + filename + " image"); } } } /* Magnifies each region of {this.reg} from {level_current} to {level_base}. * Than merges the elements of letter list {this.reg} which have {dec < 0} with {reg_base}. * Assumes that the regions in {reg_base} already magnified for {level_base}. *If two boxes overlap significantly retains only the one with better {dec} score. */ public ArrayList Merge_Letter_List (int level_base, ArrayList regions_base, int level_current) { ArrayList regions_end = new ArrayList(); assert(level_current == pyramid_level); for (int i = 0; i < regions.length; i++) { Regions.rescale_size (regions[i], level_current, level_base); } /*Take every element of {reg} with {dec < 0} that is not strictly dominated by an element of {reg_base}*/ for (int i = 0; i < regions.length; i++) { Regions regionsi = regions[i]; if (regionsi.getDec() >= 0) {continue;} boolean dominated = false; /*for (int j = 0; j < regions_base.size(); j++) { Regions regionsj = regions_base.get(j); if ( (Regions.overlap(regionsj, regionsi) > 0.3) && (Regions.dominates(regionsj, regionsi)) ) { dominated = true; } }*/ if (!dominated) { regions_end.add(regionsi); } } /*Take every element of {reg_base} that strictly dominates every element of {reg} that has {dec < 0} and overlaps it*/ for (int i = 0; i < regions_base.size(); i++) { Regions regi = regions_base.get(i); //assert(regi.getDec() < 0); boolean dominates_all = true; /*for (int j = 0; j < regions.length; j++) { Regions regj = regions[j]; if (regj.getDec() >= 0) {continue;} if ( (Regions.overlap(regj, regi) > 0.3) && (!Regions.dominates(regi, regj)) ) { dominates_all = false; } }*/ if (dominates_all) { regions_end.add(regi); } } return regions_end; } public BufferedImage Segmentation (BufferedImage image, boolean inversion, int pyramid_level, detection_arguments detection_parameters) { int w = image.getWidth(null); int h = image.getHeight(null); int size = w * h; int[] min = new int[size]; int[] max = new int[size]; int[] f_h1 = new int[size]; int[] f_h2 = new int[size]; Util util = new Util(); int[] array = util.getImageArray2 (image); Segmentation segmentation = new Segmentation(); segmentation.Min_Max (w, h, array, min, max, detection_parameters.mask_size); segmentation.Double_Toggle_Mapping (w, h, array, min, max, f_h1, f_h2, detection_parameters); BufferedImage out = Util.getImage (w, h, f_h1); if (detection_parameters.write_image) { Util.writeArrayToPGMImage (w, h, min, Util.make_name ("min", inversion, pyramid_level, ".pgm")); Util.writeArrayToPGMImage (w, h, max, Util.make_name ("max", inversion, pyramid_level, ".pgm")); Util.writeArrayToPGMImage (w, h, f_h1, Util.make_name ("thresholdedA", inversion, pyramid_level, ".pgm")); Util.writeArrayToPGMImage (w, h, f_h2, Util.make_name ("thresholdedB", inversion, pyramid_level, ".pgm")); } return out; } Labels Refining_and_Labeling (BufferedImage image_segmented, int pyramid_level, detection_arguments detection_parameters) { int w = image_segmented.getWidth(null); int h = image_segmented.getHeight(null); Util util = new Util(); int[] isegmented = util.getImageArray2(image_segmented); Labels labels = Labeling (w, h, isegmented); Write_Label_Image (labels, "labels_raw", pyramid_level, detection_parameters); Segmentation segmentation = new Segmentation(); segmentation.Remove_Unknown_Value (w, h, isegmented, labels, pyramid_level, detection_parameters); if (detection_parameters.write_image) { Util.writePGMImage(image_segmented, Util.make_name ("thresholded_wun", inversion, pyramid_level, ".pgm")); } labels = Labeling (w, h, isegmented); Write_Label_Image (labels, "labels_wun", pyramid_level, detection_parameters); segmentation.Size_Check (w, h, isegmented, labels, pyramid_level, detection_parameters); labels = Labeling (w, h, isegmented); Write_Label_Image (labels, "labels_refined", pyramid_level, detection_parameters); regions = segmentation.Analyze_Regions (w, h, isegmented, labels, pyramid_level); return labels; } public void Classification2 ( Labels labels, boolean inversion, detection_arguments detection_parameters ) { long start; double fourier[] = new double[labels.getNumberOfRegions()-1]; double polar[] = new double[labels.getNumberOfRegions()-1]; double zernike[] = new double[labels.getNumberOfRegions()-1]; boolean normalization = false; /*Fourier classification*/ if (detection_parameters.descriptors[0]) { start = System.currentTimeMillis(); Features fourier_descriptor = new Features(); SVM_DATA fourier_obj = svm_classifiers.get(0); ArrayList> list = fourier_descriptor.Extracting_Fourier2 (labels, regions, fourier_obj.mean, fourier_obj.deviation, normalization, parameters); for(int i = 1 ; i < labels.getNumberOfRegions(); i++) { TrainingSample e = list.get(i-1); /*Score found to each segmented image region*/ double v = fourier_obj.cls.valueOf(e.sample); fourier[i-1] = v; regions[i].setDec(-v); } Util.Compute_Processing_Time (start, "Fourier SVM Classification"); } /*Polar classification*/ if (detection_parameters.descriptors[1]) { start = System.currentTimeMillis(); Features polar_descriptor = new Features(); SVM_DATA polar_obj = svm_classifiers.get(1); ArrayList> list = polar_descriptor.Extracting_Polar (labels, regions, polar_obj.mean, polar_obj.deviation, normalization, parameters); //System.err.printf("Polar List size : %d\n", list.size()); for(int i = 1 ; i < labels.getNumberOfRegions(); i++) { TrainingSample e = list.get(i-1); /*Score found to each segmented image region*/ double v = polar_obj.cls.valueOf(e.sample); polar[i-1] = v; regions[i].setDec(-v); } Util.Compute_Processing_Time (start, "Polar SVM Classification"); } /*Zernike classification*/ if (detection_parameters.descriptors[2]) { start = System.currentTimeMillis(); Features zernike_descriptor = new Features(); SVM_DATA zernike_obj = svm_classifiers.get(2); ArrayList> list = zernike_descriptor.Extracting_Zernike (labels, regions, zernike_obj.mean, zernike_obj.deviation, normalization, parameters); //System.err.printf("Zernike List size : %d\n", list.size()); for(int i = 1 ; i < labels.getNumberOfRegions(); i++) { TrainingSample e = list.get(i-1); /*Score found to each segmented image region*/ double v = zernike_obj.cls.valueOf(e.sample); zernike[i-1] = v; regions[i].setDec(-v); } Util.Compute_Processing_Time (start, "Zernike SVM Classification"); } /*Combining all descriptors*/ if (detection_parameters.descriptors[3]) { start = System.currentTimeMillis(); /*for (int i = 0; i < detection_parameters.descriptors.length; i++) { if (!detection_parameters.descriptors[i]) { System.err.printf("Is not possible to compute the combination of the descriptors some of them are false"); System.exit(1); } }*/ Features all_descriptors = new Features(); SVM_DATA all_obj = svm_classifiers.get(3); ArrayList> list = all_descriptors.Combining_All (fourier, polar, zernike, all_obj.mean, all_obj.deviation, normalization, parameters); //System.err.printf("All List size : %d\n", list.size()); for(int i = 1 ; i < labels.getNumberOfRegions(); i++) { TrainingSample e = list.get(i-1); /*Score found to each segmented image region*/ double v = all_obj.cls.valueOf(e.sample); regions[i].setDec(-v); } Util.Compute_Processing_Time (start, "All SVM Classification"); } } /*Given of result of the segmentation {lab_contour}, filtered by size, for one pyramid level {param.getScale()} *applies the letter recognizer and grouping algorithm. The recognized letter boxes are left in {this.reg} and *recognized text boxes are left in {this.list} both magnified fir the current pyramid level. */ public void Classification1 ( Labels labels, boolean inversion, detection_arguments detection_parameters ) { SVM fourier = svm_descriptors.get(0); SVM polar = svm_descriptors.get(1); SVM zernike = svm_descriptors.get(2); SVM all = svm_descriptors.get(3); long start; Classify fourier_scores = new Classify(); Classify zernike_scores = new Classify(); Classify polar_scores = new Classify(); /*Fourier classification*/ if (detection_parameters.descriptors[0]) { start = System.currentTimeMillis(); Features fourier_descriptor = new Features(); fourier_descriptor.Extracting_Fourier (labels, regions, fourier.Get_Sample_Size(), detection_parameters); fourier_scores.classify_rbf (fourier, fourier_descriptor); Util.Compute_Processing_Time (start, "Fourier Classification"); for(int i = 1; i < labels.getNumberOfRegions(); i++) { regions[i].setDec(fourier_scores.Get_Score_Value(i-1)); //System.out.printf("Score svm : %f\n", fourier_scores.Get_Score_Value(i-1)); } } /*Zernike classification*/ if (detection_parameters.descriptors[1]) { start = System.currentTimeMillis(); Features zernike_descriptor = new Features(); zernike_descriptor.Extracting_Zernike (labels, regions, zernike.Get_Sample_Size(), detection_parameters); zernike_scores.classify_rbf (zernike, zernike_descriptor); Util.Compute_Processing_Time (start, "Zernike Classification"); for(int i = 1; i < labels.getNumberOfRegions(); i++) { regions[i].setDec(zernike_scores.Get_Score_Value(i-1)); } } /*Polar classification*/ if (detection_parameters.descriptors[2]) { start = System.currentTimeMillis(); Features polar_descriptor = new Features(); polar_descriptor.Extracting_Polar (labels, regions, polar.Get_Sample_Size(), detection_parameters); polar_scores.classify_rbf (polar, polar_descriptor); Util.Compute_Processing_Time (start, "Polar Classification"); for(int i = 1; i < labels.getNumberOfRegions(); i++) { regions[i].setDec(polar_scores.Get_Score_Value(i-1)); } } /*Combining all descriptors*/ if (detection_parameters.descriptors[3]) { for (int i = 0; i < detection_parameters.descriptors.length; i++) { if (!detection_parameters.descriptors[i]) { System.err.printf("Is not possible to compute the combination of the descriptors some of them are false"); System.exit(1); } } start = System.currentTimeMillis(); Features all_descriptors = new Features(); all_descriptors.Combining_All (fourier_scores, zernike_scores, polar_scores, detection_parameters); Classify final_scores = new Classify(); final_scores.classify_rbf (all, all_descriptors); Util.Compute_Processing_Time (start, "Final Classification"); /*Marking each region with the end-score of the classification*/ for(int i = 1; i < labels.getNumberOfRegions(); i++) { regions[i].setDec(final_scores.Get_Score_Value(i-1)); } } } void Compute_And_Draw_Regions_Barycenters ( BufferedImage image, BufferedImage image_segmented, Labels labels, Groupment groupment, detection_arguments detection_parameters) { long start = System.currentTimeMillis();; int w = image.getWidth(); int h = image.getHeight(); int size = w * h; Util util = new Util(); int[] array = util.getImageArray2 (image); int[] out = new int[size]; for (int i = 0; i < size; i++) { int label = labels.getValue(i); double score = regions[label].getDec(); if ( (label != 0) && (score < 0) ) { out[i] = Util.getPixelFromRGB (0, Util.Value(score), 0); } else if ( (label != 0) && (score > 0) ) { out[i] = Util.getPixelFromRGB (Util.Value(score), 0, 0); } else { if (inversion) { out[i] = Util.getPixelFromRGB (255-array[i], 255-array[i], 255-array[i]); } else { out[i] = Util.getPixelFromRGB (array[i], array[i], array[i]); } } } int startX = 0, startY = 0; BufferedImage outimg = new BufferedImage (w, h, BufferedImage.TYPE_INT_ARGB); outimg.setRGB (startX, startY, w, h, out, 0, w); groupment.Dispose_Big_Regions_Agroupments (outimg, list, detection_parameters); groupment.Dispose_Region_And_Barycenters (outimg, regions, labels.getNumberOfRegions()); groupment.Dispose_Region_And_Barycenters (image_segmented, regions, labels.getNumberOfRegions()); try { BufferedImage ilabels = labels.Get_Image_From_Labels (regions); groupment.Dispose_Region_And_Barycenters (ilabels, regions, labels.getNumberOfRegions()); ImageIO.write(ilabels, "png", new File(Util.make_name ("recognized_letters_labels_orig", inversion, pyramid_level, ".png"))); ImageIO.write(outimg, "png", new File(Util.make_name ("recognized_letters_orig", inversion, pyramid_level, ".png"))); Util.writePGMImage(image_segmented, Util.make_name ("recognized_letters_thrs", inversion, pyramid_level, ".pgm")); } catch (Exception e) { System.err.println("cannot store image"); } Util.Compute_Processing_Time (start, "Drawing barycenters"); } void Regions_Groupment ( BufferedImage image, BufferedImage image_segmented, Labels labels, detection_arguments detection_parameters) { long start; start = System.currentTimeMillis(); Groupment groupment = new Groupment(); list = groupment.Grouping_Regions (labels, regions, detection_parameters); if (parameters.write_image) { Compute_And_Draw_Regions_Barycenters (image, image_segmented, labels, groupment, detection_parameters); } Util.Compute_Processing_Time (start, "Regions_Groupment"); } public DetectionThread ( BufferedImage input_image, boolean inversion, int pyramid_level, ArrayList svm_descriptors, ArrayList svm_classifiers, detection_arguments detection_parameters) { this.input_image = input_image; this.inversion = inversion; this.pyramid_level = pyramid_level; this.parameters = detection_parameters; if (parameters.svm_option) { this.svm_descriptors = svm_descriptors; } else { this.svm_classifiers = svm_classifiers; } } public void run () { long start = System.currentTimeMillis(); BufferedImage image_segmented = Segmentation (input_image, inversion, pyramid_level, parameters); Labels labels = Refining_and_Labeling (image_segmented, pyramid_level, parameters); if (parameters.svm_option) { Classification1 (labels, inversion, parameters); } else { Classification2 (labels, inversion, parameters); } Regions_Groupment (input_image, image_segmented, labels, parameters); finished = true; Util.Compute_Processing_Time (start, "Segmentation, Classification and Grouping"); Thread.yield(); } public boolean Has_Finished() { return finished; } public LinkedList Get_Text_List() { return list; } }