001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.util.Arrays; 016 017import org.eclipse.january.DatasetException; 018import org.eclipse.january.IMonitor; 019import org.eclipse.january.metadata.StatisticsMetadata; 020import org.eclipse.january.metadata.internal.StatisticsMetadataImpl; 021import org.slf4j.Logger; 022import org.slf4j.LoggerFactory; 023 024/** 025 * Generic container class for data that is compound in nature 026 * 027 * Each subclass has an array of compound types, items of this array are composed of primitive types 028 * 029 * Data items can be Complex, Vector, etc 030 * 031 */ 032public abstract class AbstractCompoundDataset extends AbstractDataset implements CompoundDataset { 033 // pin UID to base class 034 private static final long serialVersionUID = Dataset.serialVersionUID; 035 036 private static final Logger logger = LoggerFactory.getLogger(AbstractCompoundDataset.class); 037 038 protected int isize; // number of elements per item 039 040 @Override 041 public int getElementsPerItem() { 042 return isize; 043 } 044 045 @Override 046 protected int get1DIndex(final int i) { 047 int n = super.get1DIndex(i); 048 return stride == null ? isize * n : n; 049 } 050 051 @Override 052 protected int get1DIndex(final int i, final int j) { 053 int n = super.get1DIndex(i, j); 054 return stride == null ? isize * n : n; 055 } 056 057 @Override 058 protected int get1DIndexFromShape(final int[] n) { 059 return isize * super.get1DIndexFromShape(n); 060 } 061 062 @Override 063 public Dataset getUniqueItems() { 064 throw new UnsupportedOperationException("Cannot sort compound datasets"); 065 } 066 067 @Override 068 public IndexIterator getIterator(final boolean withPosition) { 069 if (stride != null) { 070 return base.getSize() == 1 ? 071 (withPosition ? new PositionIterator(offset, shape) : new SingleItemIterator(offset, size)) : new StrideIterator(isize, shape, stride, offset); 072 } 073 return withPosition ? getSliceIterator(null, null, null) : 074 new ContiguousIterator(size, isize); 075 } 076 077 /** 078 * Get an iterator that picks out the chosen element from all items 079 * @param element to choose 080 * @return an iterator 081 */ 082 public IndexIterator getIterator(int element) { 083 if (element < 0) 084 element += isize; 085 if (element < 0 || element > isize) { 086 logger.error("Invalid choice of element: {}/{}", element, isize); 087 throw new IllegalArgumentException("Invalid choice of element: " + element + "/" + isize); 088 } 089 090 final IndexIterator it; 091 if (stride != null) { 092 it = base.getSize() == 1 ? new SingleItemIterator(offset + element, size) : new StrideIterator(isize, shape, stride, offset, element); 093 } else { 094 it = new ContiguousIterator(size, isize, element); 095 } 096 097 return it; 098 } 099 100 @Override 101 public IndexIterator getSliceIterator(SliceND slice) { 102 checkSliceND(slice); 103 return internalGetSliceIterator(slice); 104 } 105 106 /** 107 * @param slice to define iterator 108 * @return an slice iterator that operates like an IndexIterator 109 */ 110 @Override 111 protected IndexIterator internalGetSliceIterator(SliceND slice) { 112 if (ShapeUtils.calcLongSize(slice.getShape()) == 0) { 113 return new NullIterator(shape, slice.getShape()); 114 } 115 if (stride != null) { 116 return new StrideIterator(isize, shape, stride, offset, slice); 117 } 118 119 return new SliceIterator(shape, size, isize, slice); 120 } 121 122 /** 123 * Constructor required for serialisation. 124 */ 125 public AbstractCompoundDataset() { 126 } 127 128 @Override 129 public boolean equals(Object obj) { 130 if (!super.equals(obj)) { 131 return false; 132 } 133 134 CompoundDataset other = (CompoundDataset) obj; 135 return isize == other.getElementsPerItem(); 136 } 137 138 @Override 139 public int hashCode() { 140 return getCompoundStats().getHash(shape); 141 } 142 143 @Override 144 public CompoundDataset cast(boolean repeat, int dtype, int isize) { 145 return (CompoundDataset) super.cast(repeat, dtype, isize); 146 } 147 148 @Override 149 public CompoundDataset cast(int dtype) { 150 return (CompoundDataset) super.cast(dtype); 151 } 152 153 @Override 154 abstract public AbstractCompoundDataset clone(); 155 156 @Override 157 public CompoundDataset flatten() { 158 return (CompoundDataset) super.flatten(); 159 } 160 161 @Override 162 public CompoundDataset getBy1DIndex(IntegerDataset index) { 163 return (CompoundDataset) super.getBy1DIndex(index); 164 } 165 166 @Override 167 public CompoundDataset getByBoolean(Dataset selection) { 168 return (CompoundDataset) super.getByBoolean(selection); 169 } 170 171 @Override 172 public CompoundDataset getByIndexes(Object... indexes) { 173 return (CompoundDataset) super.getByIndexes(indexes); 174 } 175 176 @Override 177 public CompoundDataset getSlice(IMonitor mon, int[] start, int[] stop, int[] step) { 178 return (CompoundDataset) super.getSlice(mon, start, stop, step); 179 } 180 181 @Override 182 public CompoundDataset getSlice(IMonitor mon, Slice... slice) { 183 return (CompoundDataset) super.getSlice(mon, slice); 184 } 185 186 @Override 187 public CompoundDataset getSlice(IMonitor mon, SliceND slice) { 188 return (CompoundDataset) super.getSlice(mon, slice); 189 } 190 191 @Override 192 public CompoundDataset getSlice(int[] start, int[] stop, int[] step) { 193 return (CompoundDataset) super.getSlice(start, stop, step); 194 } 195 196 @Override 197 public CompoundDataset getSlice(Slice... slice) { 198 return (CompoundDataset) super.getSlice(slice); 199 } 200 201 @Override 202 public CompoundDataset getSlice(SliceND slice) { 203 return (CompoundDataset) super.getSlice(slice); 204 } 205 206 @Override 207 abstract public AbstractCompoundDataset getSlice(SliceIterator iterator); 208 209 @Override 210 public CompoundDataset getSliceView(int[] start, int[] stop, int[] step) { 211 return (CompoundDataset) super.getSliceView(start, stop, step); 212 } 213 214 @Override 215 public CompoundDataset getSliceView(Slice... slice) { 216 return (CompoundDataset) super.getSliceView(slice); 217 } 218 219 @Override 220 public CompoundDataset getSliceView(SliceND slice) { 221 return (CompoundDataset) super.getSliceView(slice); 222 } 223 224 @Override 225 public CompoundDataset getTransposedView(int... axes) { 226 return (CompoundDataset) super.getTransposedView(axes); 227 } 228 229 @Override 230 abstract public AbstractCompoundDataset getView(boolean deepCopyMetadata); 231 232 @Override 233 public CompoundDataset getBroadcastView(int... broadcastShape) { 234 return (CompoundDataset) super.getBroadcastView(broadcastShape); 235 } 236 237 @Override 238 public CompoundDataset ifloorDivide(Object o) { 239 return (CompoundDataset) super.ifloorDivide(o); 240 } 241 242 @Override 243 public CompoundDataset reshape(int... shape) { 244 return (CompoundDataset) super.reshape(shape); 245 } 246 247 @Override 248 public CompoundDataset setSlice(Object obj, int[] start, int[] stop, int[] step) { 249 return (CompoundDataset) super.setSlice(obj, start, stop, step); 250 } 251 252 @Override 253 public CompoundDataset setSlice(Object object, Slice... slice) { 254 return (CompoundDataset) super.setSlice(object, slice); 255 } 256 257 @Override 258 public CompoundDataset sort(Integer axis) { 259 throw new UnsupportedOperationException("Cannot sort dataset"); 260 } 261 262 @Override 263 public CompoundDataset squeezeEnds() { 264 return (CompoundDataset) super.squeezeEnds(); 265 } 266 267 @Override 268 public CompoundDataset squeeze() { 269 return (CompoundDataset) super.squeeze(); 270 } 271 272 @Override 273 public CompoundDataset squeeze(boolean onlyFromEnd) { 274 return (CompoundDataset) super.squeeze(onlyFromEnd); 275 } 276 277 @Override 278 public CompoundDataset swapAxes(int axis1, int axis2) { 279 return (CompoundDataset) super.swapAxes(axis1, axis2); 280 } 281 282 @Override 283 public synchronized CompoundDataset synchronizedCopy() { 284 return clone(); 285 } 286 287 @Override 288 public CompoundDataset transpose(int... axes) { 289 return (CompoundDataset) super.transpose(axes); 290 } 291 292 /** 293 * @since 2.0 294 * @return first value 295 */ 296 abstract protected double getFirstValue(); 297 298 abstract protected double getFirstValue(final int i); 299 300 abstract protected double getFirstValue(final int i, final int j); 301 302 abstract protected double getFirstValue(final int...pos); 303 304 @Override 305 public boolean getBoolean() { 306 return getFirstValue() != 0; 307 } 308 309 @Override 310 public boolean getBoolean(final int i) { 311 return getFirstValue(i) != 0; 312 } 313 314 @Override 315 public boolean getBoolean(final int i, final int j) { 316 return getFirstValue(i, j) != 0; 317 } 318 319 @Override 320 public boolean getBoolean(final int... pos) { 321 return getFirstValue(pos) != 0; 322 } 323 324 @Override 325 public byte getByte() { 326 return (byte) getFirstValue(); 327 } 328 329 @Override 330 public byte getByte(final int i) { 331 return (byte) getFirstValue(i); 332 } 333 334 @Override 335 public byte getByte(final int i, final int j) { 336 return (byte) getFirstValue(i, j); 337 } 338 339 @Override 340 public byte getByte(final int... pos) { 341 return (byte) getFirstValue(pos); 342 } 343 344 @Override 345 public short getShort() { 346 return (short) getFirstValue(); 347 } 348 349 @Override 350 public short getShort(final int i) { 351 return (short) getFirstValue(i); 352 } 353 354 @Override 355 public short getShort(final int i, final int j) { 356 return (short) getFirstValue(i, j); 357 } 358 359 @Override 360 public short getShort(final int... pos) { 361 return (short) getFirstValue(pos); 362 } 363 364 @Override 365 public int getInt() { 366 return (int) getFirstValue(); 367 } 368 369 @Override 370 public int getInt(final int i) { 371 return (int) getFirstValue(i); 372 } 373 374 @Override 375 public int getInt(final int i, final int j) { 376 return (int) getFirstValue(i, j); 377 } 378 379 @Override 380 public int getInt(final int... pos) { 381 return (int) getFirstValue(pos); 382 } 383 384 @Override 385 public long getLong() { 386 return (long) getFirstValue(); 387 } 388 389 @Override 390 public long getLong(final int i) { 391 return (long) getFirstValue(i); 392 } 393 394 @Override 395 public long getLong(final int i, final int j) { 396 return (long) getFirstValue(i, j); 397 } 398 399 @Override 400 public long getLong(final int... pos) { 401 return (long) getFirstValue(pos); 402 } 403 404 @Override 405 public float getFloat() { 406 return (float) getFirstValue(); 407 } 408 409 @Override 410 public float getFloat(final int i) { 411 return (float) getFirstValue(i); 412 } 413 414 @Override 415 public float getFloat(final int i, final int j) { 416 return (float) getFirstValue(i, j); 417 } 418 419 @Override 420 public float getFloat(final int... pos) { 421 return (float) getFirstValue(pos); 422 } 423 424 @Override 425 public double getDouble() { 426 return getFirstValue(); 427 } 428 429 @Override 430 public double getDouble(final int i) { 431 return getFirstValue(i); 432 } 433 434 @Override 435 public double getDouble(final int i, final int j) { 436 return getFirstValue(i, j); 437 } 438 439 @Override 440 public double getDouble(final int... pos) { 441 return getFirstValue(pos); 442 } 443 444 @Override 445 public void getDoubleArray(final double[] darray) { 446 getDoubleArrayAbs(getFirst1DIndex(), darray); 447 } 448 449 @Override 450 public void getDoubleArray(final double[] darray, final int i) { 451 getDoubleArrayAbs(get1DIndex(i), darray); 452 } 453 454 @Override 455 public void getDoubleArray(final double[] darray, final int i, final int j) { 456 getDoubleArrayAbs(get1DIndex(i, j), darray); 457 } 458 459 @Override 460 public void getDoubleArray(final double[] darray, final int... pos) { 461 getDoubleArrayAbs(get1DIndex(pos), darray); 462 } 463 464 /** 465 * @return statistics metadata 466 * @since 2.0 467 */ 468 @SuppressWarnings("unchecked") 469 protected StatisticsMetadata<double[]> getCompoundStats() { 470 StatisticsMetadata<double[]> md = getFirstMetadata(StatisticsMetadata.class); 471 if (md == null || md.isDirty(this)) { 472 md = new StatisticsMetadataImpl<double[]>(); 473 md.initialize(this); 474 setMetadata(md); 475 } 476 return md; 477 } 478 479 @Override 480 public IntegerDataset argMax(int axis, boolean... ignoreInvalids) { 481 logger.error("Cannot compare compound numbers"); 482 throw new UnsupportedOperationException("Cannot compare compound numbers"); 483 } 484 485 @Override 486 public IntegerDataset argMin(int axis, boolean... ignoreInvalids) { 487 logger.error("Cannot compare compound numbers"); 488 throw new UnsupportedOperationException("Cannot compare compound numbers"); 489 } 490 491 @Override 492 public Number max(boolean... ignoreInvalids) { 493 logger.error("Cannot compare compound numbers"); 494 throw new UnsupportedOperationException("Cannot compare compound numbers"); 495 } 496 497 @Override 498 public CompoundDataset max(int axis, boolean... ignoreInvalids) { 499 logger.error("Cannot compare compound numbers"); 500 throw new UnsupportedOperationException("Cannot compare compound numbers"); 501 } 502 503 @Override 504 public Number min(boolean... ignoreInvalids) { 505 logger.error("Cannot compare compound numbers"); 506 throw new UnsupportedOperationException("Cannot compare compound numbers"); 507 } 508 509 @Override 510 public CompoundDataset min(int axis, boolean... ignoreInvalids) { 511 logger.error("Cannot compare compound numbers"); 512 throw new UnsupportedOperationException("Cannot compare compound numbers"); 513 } 514 515 516 @Override 517 public int[] maxPos(boolean... ignoreNaNs) { 518 logger.error("Cannot compare compound numbers"); 519 throw new UnsupportedOperationException("Cannot compare compound numbers"); 520 } 521 522 @Override 523 public int[] minPos(boolean... ignoreNaNs) { 524 logger.error("Cannot compare compound numbers"); 525 throw new UnsupportedOperationException("Cannot compare compound numbers"); 526 } 527 528 @Override 529 public CompoundDataset peakToPeak(int axis, boolean... ignoreInvalids) { 530 logger.error("Cannot compare compound numbers"); 531 throw new UnsupportedOperationException("Cannot compare compound numbers"); 532 } 533 534 @Override 535 public double[] maxItem() { 536 return getCompoundStats().getMaximum(); 537 } 538 539 @Override 540 public double[] minItem() { 541 return getCompoundStats().getMinimum(); 542 } 543 544 @Override 545 public Object mean(boolean... ignoreInvalids) { 546 return getCompoundStats().getMean(); 547 } 548 549 @Override 550 public CompoundDataset mean(int axis, boolean... ignoreInvalids) { 551 return (CompoundDataset) super.mean(axis, ignoreInvalids); 552 } 553 554 @Override 555 public CompoundDataset product(int axis, boolean... ignoreInvalids) { 556 return (CompoundDataset) super.product(axis, ignoreInvalids); 557 } 558 559 @Override 560 public CompoundDataset rootMeanSquare(int axis, boolean... ignoreInvalids) { 561 return (CompoundDataset) super.rootMeanSquare(axis, ignoreInvalids); 562 } 563 564 @Override 565 public CompoundDataset stdDeviation(int axis) { 566 return (CompoundDataset) super.stdDeviation(axis, false); 567 } 568 569 @Override 570 public CompoundDataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 571 return (CompoundDataset) super.stdDeviation(axis, isWholePopulation, ignoreInvalids); 572 } 573 574 @Override 575 public Object sum(boolean... ignoreInvalids) { 576 return getCompoundStats().getSum(); 577 } 578 579 @Override 580 public CompoundDataset sum(int axis, boolean... ignoreInvalids) { 581 return (CompoundDataset) super.sum(axis, ignoreInvalids); 582 } 583 584 @Override 585 public double variance(boolean isWholePopulation, boolean... ignoreInvalids) { 586 return getCompoundStats().getVariance(isWholePopulation, ignoreInvalids); 587 } 588 589 @Override 590 public CompoundDataset variance(int axis) { 591 return (CompoundDataset) super.variance(axis, false); 592 } 593 594 @Override 595 public CompoundDataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) { 596 return (CompoundDataset) super.variance(axis, isWholePopulation, ignoreInvalids); 597 } 598 599 @Override 600 public double rootMeanSquare(boolean... ignoreInvalids) { 601 StatisticsMetadata<double[]> stats = getCompoundStats(); 602 603 double[] mean = stats.getMean(ignoreInvalids); 604 double result = 0; 605 for (int i = 0; i < isize; i++) { 606 double m = mean[i]; 607 result += m * m; 608 } 609 return Math.sqrt(result + stats.getVariance(true)); 610 } 611 612 /** 613 * @return error 614 */ 615 private CompoundDataset getInternalError() { 616 ILazyDataset led = super.getErrors(); 617 if (led == null) 618 return null; 619 620 Dataset ed = null; 621 try { 622 ed = DatasetUtils.sliceAndConvertLazyDataset(led); 623 } catch (DatasetException e) { 624 logger.error("Could not get data from lazy dataset", e); 625 } 626 627 CompoundDataset ced; // ensure it has the same number of elements 628 if (!(ed instanceof CompoundDataset) || ed.getElementsPerItem() != isize) { 629 ced = new CompoundDoubleDataset(isize, true, ed); 630 } else { 631 ced = (CompoundDataset) ed; 632 } 633 634 if (led != ced) { 635 setErrors(ced); // set back 636 } 637 return ced; 638 } 639 640 @Override 641 public CompoundDataset getErrors() { 642 CompoundDataset ed = getInternalError(); 643 if (ed == null) 644 return null; 645 646 return ed.getBroadcastView(shape); 647 } 648 649 @Override 650 public double getError(final int i) { 651 return calcError(getInternalErrorArray(true, i)); 652 } 653 654 @Override 655 public double getError(final int i, final int j) { 656 return calcError(getInternalErrorArray(true, i, j)); 657 } 658 659 @Override 660 public double getError(final int... pos) { 661 return calcError(getInternalErrorArray(true, pos)); 662 } 663 664 private double calcError(double[] es) { 665 if (es == null) 666 return 0; 667 668 // assume elements are independent 669 double e = 0; 670 for (int k = 0; k < isize; k++) { 671 e += es[k]; 672 } 673 674 return Math.sqrt(e); 675 } 676 677 @Override 678 public double[] getErrorArray(final int i) { 679 return getInternalErrorArray(false, i); 680 } 681 682 @Override 683 public double[] getErrorArray(final int i, final int j) { 684 return getInternalErrorArray(false, i, j); 685 } 686 687 @Override 688 public double[] getErrorArray(final int... pos) { 689 return getInternalErrorArray(false, pos); 690 } 691 692 private Dataset getInternalError(final boolean squared) { 693 Dataset sed = squared ? getInternalSquaredError() : getInternalError(); 694 if (sed == null) 695 return null; 696 697 return sed.getBroadcastView(shape); 698 } 699 700 private double[] getInternalErrorArray(final boolean squared, final int i) { 701 Dataset sed = getInternalError(squared); 702 if (sed == null) 703 return null; 704 705 double[] es; 706 if (sed instanceof CompoundDoubleDataset) { 707 es = ((CompoundDoubleDataset) sed).getDoubleArray(i); 708 if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted 709 Arrays.fill(es, es[0]); 710 } 711 } else { 712 es = new double[isize]; 713 Arrays.fill(es, ((DoubleDataset) sed).getDouble(i)); 714 } 715 return es; 716 } 717 718 private double[] getInternalErrorArray(final boolean squared, final int i, final int j) { 719 Dataset sed = getInternalError(squared); 720 if (sed == null) 721 return null; 722 723 double[] es; 724 if (sed instanceof CompoundDoubleDataset) { 725 es = ((CompoundDoubleDataset) sed).getDoubleArray(i, j); 726 if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted 727 Arrays.fill(es, es[0]); 728 } 729 } else { 730 es = new double[isize]; 731 Arrays.fill(es, ((DoubleDataset) sed).getDouble(i, j)); 732 } 733 return es; 734 } 735 736 private double[] getInternalErrorArray(final boolean squared, final int... pos) { 737 Dataset sed = getInternalError(squared); 738 if (sed == null) 739 return null; 740 741 double[] es = new double[isize]; 742 if (sed instanceof CompoundDoubleDataset) { 743 es = ((CompoundDoubleDataset) sed).getDoubleArray(pos); 744 if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted 745 Arrays.fill(es, es[0]); 746 } 747 } else { 748 es = new double[isize]; 749 Arrays.fill(es, ((DoubleDataset) sed).getDouble(pos)); 750 } 751 return es; 752 } 753} 754