/* Last edited on 2011-08-22 17:22:50 by stolfilocal */ package main; import java.awt.BasicStroke; import java.awt.Color; import java.awt.Font; import java.awt.Graphics2D; import java.awt.image.BufferedImage; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import javax.imageio.ImageIO; import association.HungarianAlgorithm; import association.Probabilities; import parameters.detection_arguments; import parameters.io_arguments; import preprocessing.Parser; import preprocessing.Util; import tracking.TrackRegion; import tracking.Pair; import classification.SVM; import classification.SVM_DATA; /* Criar classe que é dois conjuntos de caixas {D,E} e uma relação {phi} de {D} para {E} */ public class TrackRelation { public ArrayList L; public ArrayList R; public ArrayList P; TrackRelation() { this.L = new ArrayList(); this.R = new ArrayList(); this.P = new ArrayList(); } /* Adds the pair {(d,e)} to the relation, if not there. Also adds {d} to the left set {L}, if not there, and {e} to the right set {R}, if not there. As a special case, if {d} is null, only adds {e} to {R}. If {e} is null, only adds {d} to {L}. If both are null, does nothing. */ public addPair(TrackRegion d, TrackRegion e) { if ((d != null) && (! TrackRegionBelong(d, this.L))) { this.L.add(d); } if ((e != null) && (! TrackRegionBelong(e, this.R))) { this.R.add(e); } if ((d == null) || (e == null)) { return; } Pair p = new Pair(d, e); if (! PairBelong(p, this.P)) { this.P.add(p); } } /* Returns TRUE iff {d} occurs in {T[0..T.size-1]} */ static boolean TrackRegionBelong (TrackRegion d, ArrayList T) { for (int i = 0; i < T.size(); i++) { TrackRegion tmp = T.get(i); if (tmp == d) { return true; } } return false; } /* Returns TRUE iff there is a pair in {P[0..P.size-1]} with same left and right elements as {p}. */ static boolean PairBelong (Pair p, ArrayList P) { for (int i = 0; i < P.size(); i++) { Pair tmp = P.get(i); if ((tmp.L == p.L) && (tmp.R == p.R)) { return true; } } return false; } static TrackRegion ApplyInverseRelation (TrackRegion e, ArrayList P) { for (int i = 0; i < P.size(); i++) { TrackRegion tmp = P.get(i); if (tmp.R == e) { return tmp.L; } } return null; } } public class Strategy1 { public static void Strategy ( ArrayList svm_descriptors, ArrayList svm_classifiers, io_arguments io_parameters, detection_arguments detection_parameters ) /* Number of distinct text objects found so far: */ int num_text_objs = 0; throws FileNotFoundException, ClassNotFoundException, IOException { int bframe = io_parameters.get_bframe(); int eframe = io_parameters.get_eframe(); /* Get the HOG fingerprinting parameters: */ Parser parser = new Parser(); ArrayList HPAR = parser.Get_Array_List ("./inputs/hog_fingerprint.txt"); /* {T[i]} is the list of text regions detected in frame {bframe+i} */ ArrayList> T = new ArrayList>(); /* {P[i]} is the tracking relation between {T[i-1]} and {T[i]}. */ /* {P[0]} is always {null}. */ ArrayList> P = new ArrayList>(); /*Performing Text Detection at frame 0*/ BufferedImage frame0 = Get_Frame (bframe, io_parameters); detection_parameters = new detection_arguments(io_parameters.get_detection_parameters_file()); T.set(0, TextDetection.Detection (frame0, svm_descriptors, svm_classifiers, io_parameters, detection_parameters)); P.set(0, null); for (int iframe = bframe+1; iframe <= eframe; iframe++) { int i = iframe - bframe; /*Performing Text Detection at frame i*/ BufferedImage Vi = Get_Frame (iframe, io_parameters); BufferedImage Vj = Get_Frame (iframe-1, io_parameters); detection_parameters = new detection_arguments(io_parameters.get_detection_parameters_file()); ArrayList Td = TextDetection.Detection (Vi, svm_descriptors, svm_classifiers, io_parameters, detection_parameters); TrackRelation EX = Extrapolate(T.get(i-1), P.get(i-1), Vi.getWidth(), Vi.getHeight()); TrackRelation MX = Merge (EX, Td, Vj, Vi, HPAR, iframe); T.set(i, MX.R); P.set(i, MX.P); } Write_Detections (T, P, io_parameters); } /* Predicts the position of each region {e} from {Ti[0..T.size-1]} in the next frame, by linear extrapolation of its motion from the previous frame. Uses the tracking relation {Pi} to find the corresponding region in the previous frame. Returns {(L,R,P)} where {R} is the set of successfully extrapolated regions in the next frame, {L} are the corresponding regions in {Ti}, and {P} is the pairing between them. */ static TrackRelation Extrapolate ( ArrayList Ti, ArrayList Pi, int wd, int ht) { TrackRelation EX = new TrackRelation(); for (int j = 0; j < Ti.size(); j++) { TrackRegion e = Ti.get(j); TrackRegion d = ApplyInverseRelation (e, Pi); if (d != null) { TrackRegion f = Get_Extrapolate_Region (d, e, wd, ht); if (f != null) { EX.addPair(e,f); } } else { TrackRegion f = e.clone(); EX.addPair(e,f); } } return EX; } /* Estimates the parameters of a text region {e} in the next frame,given its parameters {d} in the previous frame. Returns {null} is the extrapolated region lies outside the next frame, whhich is assumed to have {wd} columns by {ht} rows. */ static TrackRegion Get_Extrapolate_Region (TrackRegion d, TrackRegion e, int wd, int ht) { TrackRegion f = new TrackRegion(); double[] Cd = d.get_C(); double[] Ud = d.get_U(); double[] Vd = d.get_V(); double[] Ce = e.get_C(); double[] Ue = e.get_U(); double[] Ve = e.get_V(); f.set_C(Extrapolate_Position (Ce, Cd)); f.set_U(Extrapolate_Position (Ue, Ud)); f.set_V(Extrapolate_Position (Ve, Vd)); f.set_label(e.get_label()); if (TextRegionInside(f, wd, ht)) { return f; } else { return null; } } static boolean TextRegionInside(TrackRegion f, int wd, int ht) { for (int du = -1; du <= +1; du += 2) { for (int dv = -1; dv <= +1; dv += 2) { double px = f.get_Cx() + du*f.get_Ux() + dv*f.get_Vx(); double py = f.get_Cy() + du*f.get_Uy() + dv*f.get_Vy(); if ((px < 0) || (px > wd) || (py < 0) || (py > ht)) { return false; } } } return true; } static double[] Extrapolate_Position (double[] p_2, double[] p_1) { assert ( (p_1.length == p_2.length) && (p_1.length == 2) ); double[] p = new double[2]; p[0] = 2*p_1[0] - p_2[0]; p[1] = 2*p_1[1] - p_2[1]; return p; } static TrackRelation Merge ( TrackRelation EX, ArrayList D, BufferedImage Vj, BufferedImage Vi, ArrayList HPAR, int iframe ) { TrackRelation MX = new TrackRelation(); boolean transpose = false; double assignment_threshold = 0.05; /* Using Hungarian algorithm, to create the list with the best assignments */ double[][] SM = SimilarityMatrix(EX, D, Vj, Vi, HPAR); HungarianAlgorithm HA = new HungarianAlgorithm(); if (SM.length > SM[0].length) { SM = HA.transpose(SM); transpose = true; } /*Getting the list of assignments*/ int[][] A = HA.hgAlgorithm (SM, "max"); /*-------------------------------------------*/ /*Connect those elements in {E} that have an association in {D}*/ for (int i = 0; i < A.length; i++) { int ek = A[i][0]; int dk = A[i][1]; if (P[ek][dk] > assignment_threshold) { TrackRegion Rd = null; /* Detected region. */ TrackRegion Re = null; /* Extrapolated region. */ if (transpose) { Rd = E.get(ek); Re = E.get(dk); } else { Rd = E.get(dk); Re = E.get(ek); } /* Keep the detected region {Rd}, paired to the antecedent of the extrapolated one {Re}: */ TextRegion Rf = ApplyInverseRelation (Re, EX.P); MX.addPair(Rf, Rd); Rd.set_label(Re.get_label()); num_text_objs++; } } System.out.printf("Fim do Aqui0\n"); System.out.printf("Aqui1\n"); /* Insert those elements in {D} that do not have an association in {E} as new text objects: */ for (int i = 0; i < D.size(); i++) { TrackRegion Rd = D.get (i); if (! TrackRegionBelong(Rd, MX.R)) { Rd.set_label(String.format("T:%03d", num_text_objs)); Mx.addPair(null,Rd); num_text_objs++; } } } static BufferedImage Get_Image_Region (BufferedImage image, TrackRegion R) { double[] C = R.get_C(); double[] U = R.get_U(); double[] V = R.get_V(); System.out.printf("C : %f %f\n", C[0], C[1]); System.out.printf("U : %f %f\n", U[0], U[1]); System.out.printf("V : %f %f\n", V[0], V[1]); /* Handle axis-aligned rectangles only for now: */ assert(U[1] == 0); assert(V[0] == 0); int xmin = (int)(C[0] - U[0]); int xmax = (int)(C[0] + U[0]); int ymin = (int)(C[1] - V[1]); int ymax = (int)(C[1] + V[1]); System.out.printf("xmin : %d, ymin : %d, xmax : %d, ymax : %d, width : %d, height : %d\n", xmin, ymin, xmax, ymax, image.getWidth(), image.getHeight()); BufferedImage region = Util.getSubImage(image, xmin, ymin, xmax, ymax); return region; } static double[][] SimilarityMatrix ( TrackRelation EX, ArrayList D, BufferedImage Vj, BufferedImage Vi ) { ArrayList E = EX.R; double sigma_s = 1.0; double sigma_p = 16.0; double sigma_a = 20.0; Probabilities Pr = new Probabilities(); double[][] SM = new double[E.size()][D.size()]; for (int ek = 0; ek < E.size(); ek++) { TrackRegion Re = E.get(ek); /* Extrapolated region in frame {Vi} */ TrackRegion Rf = ApplyInverseRelation (Re, EX.P); /* Corresp. region in frame {Vj} */ BufferedImage Vf = Get_Image_Region (Vj, Rf); for (int dk = 0; dk < D.size(); dk++) { TrackRegion Rd = D.get (dk); BufferedImage Vd = Get_Image_Region (Vi, Rd); double prob_s = Pr.prob_s (Re, Rd, sigma_s); double prob_p = Pr.prob_p (Re, Rd, sigma_p); double prob_a = Pr.prob_a (Vf, Vd, sigma_a, HPAR); SM[ek][dk] = prob_s * prob_p * prob_a; } } return SM; } static void Write_Detections (ArrayList> T, ArrayList> P, io_arguments io_parameters) { for (int iframe = bframe; iframe <= eframe; iframe++) { /*Creating sub-directories (for each frame) to store the detection and tracking results*/ File directory = new File (io_parameters.get_out_path() + String.format("%06d", iframe)); directory.mkdir(); /*Writing detection results*/ BufferedImage Vi = Get_Frame (iframe, io_parameters); if (iframe == bframe) { write_set_regions (Vi, T.get(iframe), null, io_parameters, false, iframe); } else { write_set_regions (Vi, T.get(iframe), P.get(iframe-1), io_parameters, false, iframe); } } } static BufferedImage Get_Frame (int iframe, io_arguments io_parameters) { /*Getting the image frame name*/ String frame_name = io_parameters.get_frames_path() + String.format("%06d", iframe) + ".png"; /*Reading the image frame*/ try { BufferedImage frame = ImageIO.read(new File(frame_name)); return frame; } catch (Exception e) { e.printStackTrace(); System.exit(1); return null; } } static void write_set_regions (BufferedImage frame, ArrayList set, ArrayList P_i_j, io_arguments io_parameters, boolean option, int iframe) { BufferedImage image = Util.imageScale (frame, 1.0); Graphics2D g = image.createGraphics(); g.setColor(Color.black); g.setStroke(new BasicStroke(2.0f)); Font f = new Font("SansSerif", Font.BOLD, 12); g.setFont(f); for (int box = 0; box < set.size(); box++) { String label = ""; TrackRegion A = set.get (box); if (P_i_j != null) { TrackRegion tmp = ApplyInverseRelation (A, P_i_j); if (tmp != null) { label = tmp.get_label()+"M"; } else { label = A.get_label(); } } else { label = A.get_label(); } double[] C = A.get_C(); double[] U = A.get_U(); double[] V = A.get_V(); int x = (int)(C[0] - V[0]); int y = (int)(C[1] - U[1]); int w = (int)(V[0]*2); int h = (int)(U[1]*2); g.drawRect(x, y, w, h); g.drawString(label, x, y); } g.dispose(); try { ImageIO.write(image, "png", new File(io_parameters.get_out_path() + String.format("%06d", iframe) + "/detection.png")); } catch (Exception e) { System.err.printf("cannot write image %s\n", io_parameters.get_out_path() + String.format("%06d", iframe) + "/detection.png"); } } }