fixed the isse with Theil-Sen Estimator

master
Armin Wolf 4 years ago
parent 77ad52ee53
commit ce49fceeee

File diff suppressed because it is too large Load Diff

@ -154,10 +154,9 @@ public class RepeatedMedianEstimator implements Algorithm, Flow.Publisher<Data>
*/
public Double estimateMedianIntersectionAbscissas(List<Line> lines, Line sampledLine) {
IntersectionCounter intersectionCounter = new IntersectionCounter();
List<Double> intersections = intersectionCounter.calculateIntersectionAbscissas(lines, sampledLine, original.getLower(), original.getUpper());
List<Double> left = intersectionCounter.calculateIntersectionAbscissas(lines, sampledLine, original.getLower(), interval.getLower());
List<Double> center = intersectionCounter.calculateIntersectionAbscissas(lines, sampledLine, interval.getLower(), interval.getUpper());
List<Double> intersections = IntersectionComputer.getInstance().calculateIntersectionAbscissas(lines, sampledLine, original.getLower(), original.getUpper());
List<Double> left = IntersectionComputer.getInstance().calculateIntersectionAbscissas(lines, sampledLine, original.getLower(), interval.getLower());
List<Double> center = IntersectionComputer.getInstance().calculateIntersectionAbscissas(lines, sampledLine, interval.getLower(), interval.getUpper());
double ki = Math.ceil((n - 1) * 0.5) - left.size();

@ -22,12 +22,10 @@ import java.util.concurrent.Flow;
*/
public class TheilSenEstimator implements Algorithm, Flow.Publisher<Data> {
private final double POSITIV_INF = 99999.0;
private final double NEGATIV_INF = -99999.0;
private final double POSITIV_INF = 9999.0;
private final double NEGATIV_INF = -9999.0;
private final double EPSILON = 0.00001;
private List<Line> setOfLines;
private List<Point> intervalIntersections;
private List<Point> sampledIntersections;
//Hilfsvariablen (siehe original Paper)
private double j;
private int jA;
@ -43,7 +41,6 @@ public class TheilSenEstimator implements Algorithm, Flow.Publisher<Data> {
private double slope;
private double yInterception;
private Flow.Subscriber<? super AlgorithmData> subscriber;
private Collection<Point> setOfIntersections;
/**
* Konstruktor
@ -54,10 +51,8 @@ public class TheilSenEstimator implements Algorithm, Flow.Publisher<Data> {
public TheilSenEstimator(List<Line> setOfLines, Presenter presenter) {
this.setOfLines = new ArrayList<>(setOfLines);
this.setOfIntersections = new HashSet<>();
this.n = setOfLines.size();
this.sampledIntersections = new ArrayList<>();
this.N = BinomialCoeffizient.run(n, 2);
//this.k = Integer.valueOf((int) (N * 0.5)) - 1;
this.k = (int) (N / 2);
@ -86,12 +81,13 @@ public class TheilSenEstimator implements Algorithm, Flow.Publisher<Data> {
//Collections.sort(intervalIntersections);
r = n;
List<Point> intervalIntersections = new LinkedList<>(IntersectionComputer.getInstance().compute(setOfLines, interval.getLower(), interval.getUpper()));
while (true) {
if (this.N <= n || (Math.abs(interval.getUpper() - interval.getLower())) < EPSILON) {
break;
} else {
//Anzahl der Schnittpunkte im Intervall [-Inf, a)
int numberOfIntersections = getOpenIntervalSize(NEGATIV_INF, interval.getLower());
int numberOfIntersections = getIntervalSize(NEGATIV_INF, interval.getLower());
//Randomized Interpolating Search
j = (r / N) * (double) (k - numberOfIntersections);
@ -104,7 +100,7 @@ public class TheilSenEstimator implements Algorithm, Flow.Publisher<Data> {
das Intrvall weniger als 11*N / sqrt(r) Elemente besitzt */
do {
//zufällige Stichprobe
sampledIntersections = RandomSampler.run(intervalIntersections, r);
List<Point> sampledIntersections = RandomSampler.run(intervalIntersections, r);
aVariant = FastElementSelector.randomizedSelect(getIntersectionAbscissas(sampledIntersections), jA);
bVariant = FastElementSelector.randomizedSelect(getIntersectionAbscissas(sampledIntersections), jB);
@ -113,7 +109,7 @@ public class TheilSenEstimator implements Algorithm, Flow.Publisher<Data> {
interval.setLower(aVariant);
interval.setUpper(bVariant);
intervalIntersections = getOpenIntervalElements(interval.getLower(), interval.getUpper());
N = getOpenIntervalSize(interval.getLower(), interval.getUpper());
N = getIntervalSize(interval.getLower(), interval.getUpper());
}
}
@ -161,25 +157,7 @@ public class TheilSenEstimator implements Algorithm, Flow.Publisher<Data> {
* @return Anzahl der Schnittpunkte im Interval [a,b)
*/
public int getIntervalSize(double a, double b) {
IntersectionCounter ic = new IntersectionCounter();
return ic.run(setOfLines, new Interval(a, b));
}
/**
* Berechne wieviele von den Schnittpunkten in dem Interval zwischen <code>a</code> und <code>b</code>
* enthalten sind.
* <p>
* Inspiriert durch:
* <url>https://stackoverflow.com/questions/136474/best-way-to-pick-a-random-subset-from-a-collection</url>
*
* @param a untere Grenze des Intervals
* @param b obere Grenze des Intrvals
* @return Anzahl der Schnittpunkte im Interval (a,b)
*/
public int getOpenIntervalSize(double a, double b) {
Collection<Point> intersections = IntersectionComputer.getInstance().compute(setOfLines, a, b);
setOfIntersections.addAll(intersections);
return intersections.size();
return getOpenIntervalElements(a,b).size();
}
/**
@ -192,16 +170,8 @@ public class TheilSenEstimator implements Algorithm, Flow.Publisher<Data> {
* @return Liste der Schnittpunkte die im Interval (a,b) vertreten sind
*/
public List<Point> getOpenIntervalElements(double a, double b) {
List<Point> list = new ArrayList<>();
for (int i = 0; i < intervalIntersections.size(); i++) {
Point x = intervalIntersections.get(i);
if ((x.getX() > a && x.getX() < b) || (Math.abs(interval.getUpper() - interval.getLower())) < EPSILON) {
list.add(x);
}
}
intervalIntersections.clear();
intervalIntersections = null;
return list;
Collection<Point> intersections = IntersectionComputer.getInstance().compute(setOfLines, a, b);
return new ArrayList<>(intersections);
}
private Line pepareResult() {
@ -217,11 +187,11 @@ public class TheilSenEstimator implements Algorithm, Flow.Publisher<Data> {
List<Double> yCoords = new ArrayList<>();
for (Point p : setOfIntersections) {
for (Point p : getOpenIntervalElements(interval.getLower(), interval.getUpper())) {
yCoords.add(p.getY());
}
double pseudoIndex = getOpenIntervalSize(NEGATIV_INF, interval.getLower()) * 1.0;
double pseudoIndex = getIntervalSize(NEGATIV_INF, interval.getLower()) * 1.0;
m = FastElementSelector.randomizedSelect(resultAbscissas, k - pseudoIndex);
Set<Double> unique = new LinkedHashSet<>(yCoords);

@ -1,10 +1,8 @@
package de.wwwu.awolf.presenter.util;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import de.wwwu.awolf.model.Line;
import de.wwwu.awolf.model.Point;
import org.jfree.util.Log;
import java.util.*;
import java.util.concurrent.ForkJoinPool;
@ -47,13 +45,13 @@ public class IntersectionComputer {
@Override
protected Collection<Point> compute() {
if (this.lines.isEmpty()){
if (this.lines.isEmpty()) {
return Collections.emptyList();
} else if (this.lines.size() > THRESHOLD) {
Logging.logDebug("Bigger than threshold, split into subtask.");
return ForkJoinTask.invokeAll(createSubTask()).stream().map(ForkJoinTask::join).flatMap(Collection::stream).collect(Collectors.toList());
} else {
return work(this.lines, this.lines, this.lower, this.upper);
return work(this.fullList, this.lines, this.lower, this.upper);
}
}
@ -79,7 +77,6 @@ public class IntersectionComputer {
}
}
@ -93,11 +90,37 @@ public class IntersectionComputer {
* @return Liste der Schnittpunkte
*/
public Collection<Point> compute(final List<Line> lines, final double lower, final double higher) {
Logging.logInfo("Open ForkJoinPool: lines: " + lines.size() + " I(" + lower +", " +higher + "]");
ForkJoinPool pool = ForkJoinPool.commonPool();
RecursiveComputationTask recursiveComputationTask = new RecursiveComputationTask(lines, lines, lower, higher);
pool.execute(recursiveComputationTask);
return recursiveComputationTask.join();
if (lower == higher) {
return Collections.emptyList();
} else {
Logging.logDebug("Open ForkJoinPool: lines: " + lines.size() + " I(" + lower + ", " + higher + "]");
ForkJoinPool pool = ForkJoinPool.commonPool();
RecursiveComputationTask recursiveComputationTask = new RecursiveComputationTask(lines, lines, lower, higher);
pool.execute(recursiveComputationTask);
return recursiveComputationTask.join();
}
}
/**
* Berechnet die Schnittpunkte zwischen einer gegebenen Gerade und einer Menge an Geraden.
*
* @param set Menge an Geraden
* @param sampledLine eine spezielle Gerade
* @return Liste mit x Koordinaten der Schnittpunkte
*/
public List<Double> calculateIntersectionAbscissas(List<Line> set, Line sampledLine, double lower, double upper) {
List<Line> lines = new LinkedList<>(set);
Set<Double> intersections = new HashSet<>();
for (Line line : lines) {
if (line != sampledLine) {
if (sampledLine.doIntersect(line, lower, upper)){
intersections.add(sampledLine.intersect(line).getX());
}
}
}
return new ArrayList<>(intersections);
}
public static IntersectionComputer getInstance() {

@ -1,252 +0,0 @@
package de.wwwu.awolf.presenter.util;
import de.wwwu.awolf.model.Interval;
import de.wwwu.awolf.model.Line;
import de.wwwu.awolf.model.Pair;
import de.wwwu.awolf.model.Point;
import de.wwwu.awolf.presenter.util.Comparators.YOrderLineComparatorBegin;
import de.wwwu.awolf.presenter.util.Comparators.YOrderLineComparatorEnd;
import java.util.*;
/**
* Implementierung verschiedener Algorithmen zur Berechnung von Ausgleichsgeraden.
*
* @Author: Armin Wolf
* @Email: a_wolf28@uni-muenster.de
* @Date: 18.06.2017.
*/
public class IntersectionCounter {
private HashMap<Integer, Integer> dictionaryTO;
private HashMap<Integer, Integer> dictionaryBACK;
private ArrayList<Integer> substituted;
private ArrayList<Pair> inversions;
private List<Line> set;
//indexieren der Punkte damit die schnittpunkte berechnet werden können
private HashMap<Line, Integer> secondaryDictionaryTO;
private HashMap<Integer, Line> secondaryDictionaryBACK;
private ArrayList<Line> umin;
/**
* Berechnet die Inversionen zwischen zwei Listen mit Integer Werten. Diese Methode dient als
* Wrapper Methode. Die Logik steht in der <code>countInversions</code> Funktion.
*
* @param a Liste
* @param b Liste
* @return Anzahl an Inversionen
*/
public int run(List<Integer> a, List<Integer> b) {
dictionaryTO = new HashMap<>();
dictionaryBACK = new HashMap<>();
substituted = new ArrayList<>();
inversions = new ArrayList<>();
ArrayList<Integer> temp = new ArrayList<>();
temp.addAll(a);
for (int i = 0; i < a.size(); i++) {
dictionaryTO.put(a.get(i), i + 1);
dictionaryBACK.put(i + 1, a.get(i));
}
for (int j = 0; j < b.size(); j++) {
substituted.add(dictionaryTO.get(b.get(j)));
}
int ret = countInversions(substituted, 0, substituted.size() - 1, temp);
return ret;
}
/**
* Wrapper Methode um herauszufinden wieviele Inversionen zwischen den Schnittpunkten der Werte
* in der Liste und den Endpunkten des Intervalls entstehen.
*
* @param set Liste mit Werten (m,b) um Schnittpunkte zu berechnen
* @param interval Interval
* @return Anzahl an Inversionen
*/
public int run(List<Line> set, Interval interval) {
ArrayList<Integer> listA = new ArrayList<>();
ArrayList<Integer> listB = new ArrayList<>();
prepareData(set, interval, listA, listB);
return run(listA, listB);
}
/**
* Methode die, die Daten für die Funktion <code>run</code> vorbereitet. Es werden die Schnittpunkte
* bzgl. der unteren und oberen Grenze des Intervals und den Werten der Liste (m,b) berechnet. Diese
* Werte haben die selbe x Koordinate aber verschiedene y Koordinaten.
*
* @param set Liste mit Werten m,b
* @param interval Interval
* @param listA Schnittpunkte bzgl. unteren Grenze
* @param listB Schnittpunkte bzgl. oberen Grenze
*/
private void prepareData(List<Line> set, Interval interval, ArrayList<Integer> listA,
ArrayList<Integer> listB) {
secondaryDictionaryTO = new HashMap<>();
secondaryDictionaryBACK = new HashMap<>();
this.set = set;
umin = new ArrayList<>();
Line tmpLine;
for (Line p : set) {
//vertauscht das Point standardmäßig die x lexikografische Ordnung betrachtet
tmpLine = new Line(p.getM(), p.getM(), interval.getLower() * p.getM() + p.getB(),
interval.getUpper() * p.getM() + p.getB());
//wird benötigt um späer die Schnittpunkte ermitteln zu können
tmpLine.setB(p.getB());
tmpLine.setM(p.getM());
umin.add(tmpLine);
}
for (int i = 0; i < umin.size(); i++) {
secondaryDictionaryTO.put(umin.get(i), i);
secondaryDictionaryBACK.put(i, this.set.get(i));
}
Collections.sort(umin, new YOrderLineComparatorBegin());
for (Line q : umin) {
listA.add(secondaryDictionaryTO.get(q));
}
Collections.sort(umin, new YOrderLineComparatorEnd());
for (Line q : umin) {
listB.add(secondaryDictionaryTO.get(q));
}
}
/**
* Angepasster Merge-Sort Algorithmus.
* Die Funktion bekommt neben den standard Parametern zusätzlich eine Liste mit Elementen
* die als Groundtruth dienen.
*
* @param a Eingabefeld mit den Elementen die überprüft werden sollen.
* @param start Startpunkt des Eingabefeldes.
* @param end Endpunkt des Eingabefeldes.
* @param aux Temporäres Array das beim Mergen eine Kopie des original Arrays ist.
* @return Anzahl der inversionen zwischen a und aux.
*/
public int countInversions(List<Integer> a, int start, int end, List<Integer> aux) {
if (start >= end) {
return 0;
}
int invCount = 0;
int mid = start + (end - start) / 2;
int invCountLeft = countInversions(a, start, mid, aux); // divide and conquer
int invCountRight = countInversions(a, mid + 1, end, aux); // divide and conquer
invCount += (invCountLeft + invCountRight);
for (int i = start; i <= end; i++) {
aux.set(i, a.get(i));
}
int left = start;
int right = mid + 1;
int index = start;
//hier beginnt das merging
//iteriere über die Teillisten
while (left <= mid && right <= end) {
//wenn die linke Teilliste das kleinere Element besitzt kopiere
//das Element in das neue Array
if (aux.get(left) < aux.get(right)) {
a.set(index++, aux.get(left++));
} else {
//merke die inversionspaare
for (int i = left; i <= mid; i++) {
// Logging.logInfo(aux.get(i)+" -- "+ aux.get(right));
inversions.add(new Pair(aux.get(i), aux.get(right)));
}
a.set(index++, aux.get(right++));
invCount += mid - left + 1; // number of inversions for aux[right]
}
}
while (left <= mid) {
a.set(index++, aux.get(left++));
}
// no need to copy over remaining aux[right++] because they are already inside a
return invCount;
}
/**
* Diese Methode liefert nur nach dem Ausführen der <code>run</code> Funktion Sinnvolle Werte.
*/
public ArrayList<Point> calculateIntersectionAbscissas() {
ArrayList<Pair> result = new ArrayList<>();
ArrayList<Point> points = new ArrayList<>();
for (int i = 0; i < inversions.size(); i++) {
result.add(new Pair(dictionaryBACK.get(inversions.get(i).getP1()),
dictionaryBACK.get(inversions.get(i).getP2())));
}
for (Pair p : result) {
Line line = secondaryDictionaryBACK.get(p.getP1());
Line sampledLine = secondaryDictionaryBACK.get(p.getP2());
if (!line.equals(sampledLine)) {
double intersection = (line.getB() - sampledLine.getB()) / (sampledLine.getM() - line.getM());
double yintercept = sampledLine.getM() * intersection + sampledLine.getB();
Point point = new Point(intersection, yintercept);
points.add(point);
}
}
return points;
}
/**
* Berechnet die Schnittpunkte zwischen einer gegebenen Gerade und einer Menge an Geraden.
*
* @param set Menge an Geraden
* @param sampledLine eine spezielle Gerade
* @return Liste mit x Koordinaten der Schnittpunkte
*/
public List<Double> calculateIntersectionAbscissas(List<Line> set, Line sampledLine, double lower, double upper) {
List<Line> lines = new LinkedList<>(set);
Set<Double> intersections = new HashSet<>();
for (Line line : lines) {
if (line != sampledLine) {
if (sampledLine.doIntersect(line, lower, upper)){
intersections.add(sampledLine.intersect(line).getX());
}
}
}
return new ArrayList<>(intersections);
}
/**
* Berechnet die Schnittpunkte zwischen einer gegebenen Gerade und einer Menge an Geraden.
*
* @param set Menge an Geraden
* @param sampledLine eine spezielle Gerade
* @return Liste mit x Koordinaten der Schnittpunkte
*/
public List<Double> calculateIntersectionYInterception(List<Line> set, Line sampledLine, double lower, double upper) {
List<Line> lines = new LinkedList<>(set);
Set<Double> intersections = new HashSet<>();
for (Line line : lines) {
if (line != sampledLine) {
if (sampledLine.doIntersect(line, lower, upper)){
intersections.add(sampledLine.intersect(line).getY());
}
}
}
return new ArrayList<>(intersections);
}
}

@ -1,11 +1,8 @@
package de.wwwu.awolf.presenter.util;
import de.wwwu.awolf.model.Line;
import de.wwwu.awolf.model.Point;
import java.io.Serializable;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
@ -31,13 +28,17 @@ public class RandomSampler {
* @return <code>r</code> Elementige zufällige Stichprobe an Geraden
*/
public static <T extends Object> List<T> run(List<T> set, Double r) {
int index = 0;
List<T> sampled = new ArrayList<>();
for (int i=0; i<r; i++) {
index = random.nextInt(set.size());
sampled.add(set.get(index));
if (set.isEmpty()) {
return Collections.emptyList();
} else {
int index = 0;
List<T> sampled = new ArrayList<>();
for (int i = 0; i < r; i++) {
index = random.nextInt(set.size());
sampled.add(set.get(index));
}
return sampled;
}
return sampled;
}
}

@ -19,12 +19,11 @@ import static org.junit.Assert.*;
*/
public class IntersectionCounterTest {
private IntersectionCounter intersectionCounter;
private IntersectionComputer intersectionComputer = IntersectionComputer.getInstance();
private LineModel lineModel;
@Before
public void setUp() throws Exception {
intersectionCounter = new IntersectionCounter();
lineModel = new LineModel();
lineModel.addLine(new Line(3,13,10,3));
lineModel.addLine(new Line(1,9,1,9));
@ -36,28 +35,7 @@ public class IntersectionCounterTest {
@Test
public void run() throws Exception {
assertEquals(3, intersectionCounter.run(lineModel.getLines(), new Interval(-9999,9999)));
}
@Test
public void testInversionInLists(){
// double[] umin = {6,3,4,1,2,5};
// double[] umax = {3,5,2,6,1,4};
double[] umin = {1, 2, 3, 4};
double[] umax = {2, 3, 4, 1};
ArrayList<Integer> a = new ArrayList<>();
ArrayList<Integer> b = new ArrayList<>();
for (double d : umin) {
a.add((int) d);
}
for (double d : umax) {
b.add((int) d);
}
IntersectionCounter invCounter = new IntersectionCounter();
int ret = invCounter.run(a, b);
assertEquals(3d, ret, 0.001);
assertEquals(3, intersectionComputer.compute(lineModel.getLines(), -9999,9999).size());
}
}
Loading…
Cancel
Save