Data Loaders

Note

This section does not apply to the high-level API v2.

The Location Library loads the tiles it requires using the TileLoader interface, which is how the library models versioned layers that use the HERE tiling scheme.

To obtain a TileLoader instance, select a layer name from the Catalog interface, which allows you to create a TileLoader with a layer name.

For example, to fetch the raw data of a tile from the mapping layer of the Optimized Map for Location Library, use the following:

Scala
Java
import com.here.platform.location.dataloader.core.{Catalog, TileLoader}
import com.here.platform.location.dataloader.standalone.StandaloneCatalogFactory

val factory = new StandaloneCatalogFactory
try {
  val optimizedMap: Catalog = factory.create(optimizedMapHRN, optimizedMapVersion)
  val tileLoader: TileLoader[Array[Byte]] = optimizedMap.create("mapping")
  val bytes: Option[Array[Byte]] = tileLoader.get(tileId)
  println(s"The tile size is ${bytes.get.length} bytes")
} finally {
  factory.terminate()
}
import com.here.platform.location.dataloader.core.Catalog;
import com.here.platform.location.dataloader.core.TileLoader;
import com.here.platform.location.dataloader.standalone.StandaloneCatalogFactory;
import scala.Option;
final StandaloneCatalogFactory factory = new StandaloneCatalogFactory();
try {
  final Catalog optimizedMap = factory.create(optimizedMapHRN, optimizedMapVersion);
  final TileLoader<byte[]> tileLoader = optimizedMap.create("mapping");
  final Option<byte[]> bytes = tileLoader.get(tileId);
  System.out.println("The tile size is " + bytes.get().length + " bytes");
} finally {
  factory.terminate();
}

Note the following:

You can also get an iterator containing all the tile IDs that exist in the layer the data loader accesses. For example, to count the number of tiles with data in the geometries layer of the Optimized Map for Location Library, use the following:

Scala
Java
import com.here.platform.location.dataloader.core.{Catalog, TileLoader}
import com.here.platform.location.dataloader.standalone.StandaloneCatalogFactory

val factory = new StandaloneCatalogFactory
try {
  val optimizedMap: Catalog = factory.create(optimizedMapHRN, optimizedMapVersion)
  val tileLoader: TileLoader[Array[Byte]] = optimizedMap.create("geometry")
  val tileIds: Iterator[TileId] = tileLoader.partitionIds
  println(s"The ${tileLoader.layerName} layer contains ${tileIds.length} tiles")
} finally {
  factory.terminate()
}
import com.here.platform.location.dataloader.core.Catalog;
import com.here.platform.location.dataloader.core.TileLoader;
import com.here.platform.location.dataloader.standalone.StandaloneCatalogFactory;
import com.here.platform.location.inmemory.geospatial.TileId;
import scala.collection.Iterator;
final StandaloneCatalogFactory factory = new StandaloneCatalogFactory();
try {
  final Catalog optimizedMap = factory.create(optimizedMapHRN, optimizedMapVersion);
  final TileLoader<byte[]> tileLoader = optimizedMap.create("geometry");
  final Iterator<TileId> tileIds = tileLoader.partitionIds();
  System.out.println(
      "The " + tileLoader.layerName() + " layer contains " + tileIds.size() + " tiles");
} finally {
  factory.terminate();
}

Caching TileLoader

For lower cost and better performance, avoid repeated download of tiles. To address this, the Location Library provides you with a class called LRUCachingTileLoader, a specialized TileLoader that is able to retain a configurable number of tiles in a least-recently-used manner, as shown below:

Scala
Java
import com.here.platform.location.dataloader.core.TileLoader
import com.here.platform.location.dataloader.core.caching._

val tileLoader: TileLoader[Array[Byte]] = optimizedMap.create("mapping")
val numberOfTilesToRetain: Long = 8
val cachingTileLoader: CachingTileLoader[Array[Byte]] =
  new LRUCachingTileLoader(tileLoader, numberOfTilesToRetain)
val bytes: Option[Array[Byte]] = cachingTileLoader.get(tileId)
val cachedBytes: Option[Array[Byte]] = cachingTileLoader.get(tileId)
import com.here.platform.location.dataloader.core.TileLoader;
import com.here.platform.location.dataloader.core.caching.CachingTileLoader;
import com.here.platform.location.dataloader.core.caching.LRUCachingTileLoader;
import scala.Option;
final TileLoader<byte[]> tileLoader = optimizedMap.create("mapping");
final long numberOfTilesToRetain = 8;
final CachingTileLoader<byte[]> cachingTileLoader =
    new LRUCachingTileLoader<>(tileLoader, numberOfTilesToRetain);
final Option<byte[]> bytes = cachingTileLoader.get(tileId);
final Option<byte[]> cachedBytes = cachingTileLoader.get(tileId);

The CachingTileLoader used in the snippet is a TileLoader that also provides caching.

Transforming TileLoader

To be able to use the raw data from the snippets above, you need to interpret it using protobuf schema the mapping layer, which means converting the data into a MappingPartition instance. To achieve this, Location Library provides you with the TransformingTileLoader class, which applies a transformation function to tiles returned by another TileLoader.

Using this class, you can implement the transformation to a MappingPartition as follows:

Scala
Java
import com.here.platform.location.dataloader.core._
import com.here.platform.pb.location.optimizedmap.mapping.v2.mapping.MappingPartition

val tileLoader: TileLoader[Array[Byte]] = optimizedMap.create("mapping")
def transformingFunction(bytes: Array[Byte]): MappingPartition =
  MappingPartition.parseFrom(bytes)
val transformingTileLoader: TileLoader[MappingPartition] =
  new TransformingTileLoader(tileLoader, transformingFunction)
val mappingPartition: Option[MappingPartition] = transformingTileLoader.get(tileId)

mappingPartition.foreach(_.mapping.take(3).zipWithIndex.foreach {
  case (hmcReference, vertexIndex) => println(s"$vertexIndex -> $hmcReference")
})
import com.google.protobuf.InvalidProtocolBufferException;
import com.here.platform.location.dataloader.core.TileLoader;
import com.here.platform.location.dataloader.core.javadsl.TransformingTileLoader;
import com.here.platform.pb.location.optimizedmap.mapping.v2.Mapping.MappingPartition;
import scala.Option;
final TileLoader<byte[]> tileLoader = optimizedMap.create("mapping");
final TileLoader<MappingPartition> transformingTileLoader =
    new TransformingTileLoader<>(
        tileLoader,
        data -> {
          try {
            return MappingPartition.parseFrom(data);
          } catch (final InvalidProtocolBufferException e) {
            throw new RuntimeException(e);
          }
        });
final Option<MappingPartition> maybeMappingPartition = transformingTileLoader.get(tileId);
if (maybeMappingPartition.isDefined()) {
  final MappingPartition mappingPartition = maybeMappingPartition.get();
  for (int vertexIndex = 0;
      vertexIndex < Math.min(3, mappingPartition.getMappingCount());
      vertexIndex++) {
    System.out.println("vertexIndex: " + vertexIndex);
    System.out.println(mappingPartition.getMapping(vertexIndex));
  }
}

CacheManager

The CacheManager, which is used extensively in the High-Level API, provides a way to do the following:

  • hold the created CachingTileLoader (inside a CachingTileLoaderMap ).
  • customize which kind of CachingTileLoader to create (by implementing the CachingPolicy interface).

The easiest way to create a CacheManager is through the factory method CacheManager.withLruCache(), which internally creates LRUCachingTileLoader instances by means of InMemoryLruCachePolicy. This method has a couple of overloads, which allows to set the maximum number of entries for every CacheLevel, for example:

Scala
Java
import com.here.platform.location.dataloader.core.caching.CacheManager

val maxByteArrayEntryCount = 20L
val maxInMemoryEntryCount = 10L
val maxOnTheFlyEntryCount = 30L
val cacheManager = CacheManager.withLruCache(maxByteArrayEntryCount,
                                             maxInMemoryEntryCount,
                                             maxOnTheFlyEntryCount)
import com.here.platform.location.dataloader.core.caching.CacheManager;
final long maxByteArrayEntryCount = 20;
final long maxInMemoryEntryCount = 10;
final long maxOnTheFlyEntryCount = 30;
final CacheManager cacheManager =
    CacheManager.withLruCache(
        maxByteArrayEntryCount, maxInMemoryEntryCount, maxOnTheFlyEntryCount);

CacheKey

Each TileLoader can be retrieved using a CacheKey, which is composed of the HRN and version identifying the catalog, the layer name and the CacheLevel.
The CacheLevel categorizes a TileLoader depending on the kind of data the TileLoader returns, which can be either raw bytes, decoded data, or on-the-fly compiled data.

Scala
Java
import com.here.platform.location.dataloader.core.TileLoader
import com.here.platform.location.dataloader.core.caching.{CacheKey, CacheLevel}

val tileLoader: TileLoader[Array[Byte]] = optimizedMap.create("mapping")
val cacheKey = CacheKey(optimizedMap, tileLoader.layerName, CacheLevel.ByteArray)
println(cacheKey)
import com.here.platform.location.dataloader.core.TileLoader;
import com.here.platform.location.dataloader.core.caching.CacheKey;
import com.here.platform.location.dataloader.core.caching.javadsl.CacheLevels;
final TileLoader<byte[]> tileLoader = optimizedMap.create("mapping");
final CacheKey cacheKey =
    CacheKey.apply(optimizedMap, tileLoader.layerName(), CacheLevels.BYTE_ARRAY);
System.out.println(cacheKey);

OnTheFly Cache Level

Use the on-the-fly cache level when retaining the tile loaders for on the fly compiled attributes, which are the one coming from PropertyMaps.rangeBasedAttributes and its variants (roadAttribute, navigationAttribute and advancedNavigationAttribute).
The example below shows how to create an on-they-fly tile loader whose key is a derivative of an optimized map catalog one, with the layer name equal to the chosen compiledLayerId and cache level set to OnTheFly:

Scala
Java
import com.here.platform.location.compilation.heremapcontent.AttributeAccessors
import com.here.platform.location.core.graph.RangeBasedProperty
import com.here.platform.location.dataloader.core.caching.{CacheKey, CacheLevel, CacheManager}
import com.here.platform.location.integration.optimizedmap.graph.PropertyMaps
import com.here.schema.rib.v2.road_attributes.FunctionalClassAttribute
import com.here.schema.rib.v2.road_attributes.FunctionalClassAttribute.FunctionalClass
import com.here.schema.rib.v2.road_attributes_partition.RoadAttributesPartition

val cacheManager = CacheManager.withLruCache()

val functionalClassAccessor =
  AttributeAccessors
    .forHereMapContentSegmentAnchor[RoadAttributesPartition,
                                    FunctionalClassAttribute,
                                    FunctionalClass](
      _.functionalClass,
      _.functionalClass
    )

val compiledLayerId = "on-the-fly-compiled-function-class"

val functionalClassPropertyMap =
  PropertyMaps.roadAttribute(optimizedMap,
                             compiledLayerId,
                             optimizedMap.resolveDependency(hereMapContentHRN),
                             cacheManager,
                             functionalClassAccessor)

val functionalClass: Seq[RangeBasedProperty[FunctionalClass]] =
  functionalClassPropertyMap(vertex)
println(s"Functional classes for $vertex are (${functionalClass.map(_.value).mkString(",")})")

val onTheFlyCacheKeys =
  cacheManager.loaders.stats.keys.filter(_.cacheLevel == CacheLevel.OnTheFly)

val cacheKey = CacheKey(optimizedMap, compiledLayerId, CacheLevel.OnTheFly)

assert(cacheKey == onTheFlyCacheKeys.head)
import com.here.platform.location.compilation.heremapcontent.javadsl.AttributeAccessor;
import com.here.platform.location.compilation.heremapcontent.javadsl.AttributeAccessors;
import com.here.platform.location.core.graph.javadsl.RangeBasedProperty;
import com.here.platform.location.core.graph.javadsl.RangeBasedPropertyMap;
import com.here.platform.location.dataloader.core.caching.CacheKey;
import com.here.platform.location.dataloader.core.caching.CacheManager;
import com.here.platform.location.dataloader.core.caching.javadsl.CacheLevels;
import com.here.platform.location.inmemory.graph.Vertex;
import com.here.platform.location.integration.optimizedmap.graph.javadsl.PropertyMaps;
import com.here.schema.rib.v2.road_attributes.FunctionalClassAttribute;
import com.here.schema.rib.v2.road_attributes_partition.RoadAttributesPartition;

import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static scala.collection.JavaConverters.mapAsJavaMapConverter;
final CacheManager cacheManager = CacheManager.withLruCache();
final AttributeAccessor<RoadAttributesPartition, FunctionalClassAttribute.FunctionalClass>
    functionalClassAccessor =
        AttributeAccessors.forHereMapContentSegmentAnchor(
            RoadAttributesPartition::functionalClass,
            FunctionalClassAttribute::functionalClass);

final String compiledLayerId = "on-the-fly-compiled-function-class";

final RangeBasedPropertyMap<Vertex, FunctionalClassAttribute.FunctionalClass>
    functionalClassPropertyMap =
        PropertyMaps.roadAttribute(
            optimizedMap,
            compiledLayerId,
            optimizedMap.resolveDependency(hereMapContentHRN),
            cacheManager,
            functionalClassAccessor);

final List<RangeBasedProperty<FunctionalClassAttribute.FunctionalClass>> functionalClass =
    functionalClassPropertyMap.get(vertex);
System.out.println(
    String.format(
        "Functional classes for %s are (%s)",
        vertex,
        functionalClass.stream()
            .map(fc -> fc.value().toString())
            .collect(Collectors.joining(","))));

final Stream<CacheKey> onTheFlyCacheKeys =
    mapAsJavaMapConverter(cacheManager.loaders().stats()).asJava().keySet().stream()
        .filter(k -> k.cacheLevel().equals(CacheLevels.ON_THE_FLY));

final CacheKey cacheKey = CacheKey.apply(optimizedMap, compiledLayerId, CacheLevels.ON_THE_FLY);

assertEquals(cacheKey, onTheFlyCacheKeys.findFirst().get());

The constant hereMapContentHRN is defined in the section Catalogs HRNs and versions.

Note

If you access multiple on-the-fly compiled attributes coming from the same HERE Map Content layer, we suggest you to put a value greater than zero for the ByteArray cache level size.

By doing so the HERE Map Content partitions will be reused for multiple on-the-fly attribute compilations, rather than re-downloaded after each compilation.

CachingStats

If you want to see statistics about the cache usage of a specific TileLoader, you can retrieve them through the CachingTileLoader.stats method. You could also get a global overview of cache usage as follows:

Scala
Java
import com.here.platform.location.core.graph.PropertyMap
import com.here.platform.location.dataloader.core.caching._
import com.here.platform.location.inmemory.graph.Vertex
import com.here.platform.location.integration.optimizedmap.geospatial.HereMapContentReference
import com.here.platform.location.integration.optimizedmap.graph.PropertyMaps

val cacheManager = CacheManager.withLruCache()

val vertexToHmc: PropertyMap[Vertex, HereMapContentReference] =
  PropertyMaps.vertexToHereMapContentReference(optimizedMap, cacheManager)
val hmc: HereMapContentReference = vertexToHmc(vertex)

val cacheMissesForMappingLayer: Map[CacheLevel, Long] =
  cacheManager.loaders.stats.collect {
    case (CacheKey(_, _, "mapping", level), stats) => level -> stats.missCount
  }

assert(cacheMissesForMappingLayer(CacheLevel.ByteArray) == 1)
assert(cacheMissesForMappingLayer(CacheLevel.InMemory) == 1)
import com.here.platform.location.core.graph.javadsl.PropertyMap;
import com.here.platform.location.dataloader.core.caching.CacheLevel;
import com.here.platform.location.dataloader.core.caching.CacheManager;
import com.here.platform.location.dataloader.core.caching.javadsl.CacheLevels;
import com.here.platform.location.inmemory.graph.Vertex;
import com.here.platform.location.integration.optimizedmap.geospatial.HereMapContentReference;
import com.here.platform.location.integration.optimizedmap.graph.javadsl.PropertyMaps;

import java.util.Map;
import java.util.stream.Collectors;

import static scala.collection.JavaConverters.mapAsJavaMapConverter;
final CacheManager cacheManager = CacheManager.withLruCache();
final PropertyMap<Vertex, HereMapContentReference> vertexToHmc =
    PropertyMaps.vertexToHereMapContentReference(optimizedMap, cacheManager);
final HereMapContentReference hmc = vertexToHmc.get(vertex);

final Map<CacheLevel, Long> cacheMissesForMappingLayer =
    mapAsJavaMapConverter(cacheManager.loaders().stats()).asJava().entrySet().stream()
        .filter(e -> e.getKey().layerName().equals("mapping"))
        .collect(Collectors.toMap(e -> e.getKey().cacheLevel(), e -> e.getValue().missCount()));

assertEquals(1, (long) cacheMissesForMappingLayer.get(CacheLevels.BYTE_ARRAY));
assertEquals(1, (long) cacheMissesForMappingLayer.get(CacheLevels.IN_MEMORY));

The example uses PropertyMaps to create a high-level view of the mapping layer.

CachingPolicy

If you need more flexibility on top of CacheManager.withLruCache, write your own CachingPolicy.
Suppose that you want to have an LRUCachingTileLoader with a certain cache size for a specific layer along with its CacheLevel, you could do it as follows:

Scala
Java
import com.here.platform.location.dataloader.core.TileLoader
import com.here.platform.location.dataloader.core.caching._

val cacheManager = CacheManager.withCachingPolicy(new CachingPolicy {
  override def createCachingTileLoader[T](cacheLevel: CacheLevel,
                                          inner: TileLoader[T]): CachingTileLoader[T] =
    (inner.layerName, cacheLevel) match {
      case ("mapping", CacheLevel.InMemory) =>
        new LRUCachingTileLoader[T](inner, size = 10)
      case ("mapping", CacheLevel.ByteArray) =>
        new LRUCachingTileLoader[T](inner, size = 100)
      case _ => new LRUCachingTileLoader[T](inner, size = 5)
    }
  override def createCachingDataLoader[P, T](cacheLevel: CacheLevel,
                                             inner: DataLoader[P, T]): CachingDataLoader[P, T] =
    new LRUCachingDataLoader(inner, 5)
})
import com.here.platform.location.dataloader.core.DataLoader;
import com.here.platform.location.dataloader.core.TileLoader;
import com.here.platform.location.dataloader.core.caching.*;
import com.here.platform.location.dataloader.core.caching.javadsl.CacheLevels;
CacheManager.withCachingPolicy(
    new CachingPolicy() {

      private static final long serialVersionUID = 1L;

      @Override
      public <T> CachingTileLoader<T> createCachingTileLoader(
          final CacheLevel cacheLevel, final TileLoader<T> inner) {
        if (inner.layerName().equals("mapping")) {
          if (cacheLevel.equals(CacheLevels.IN_MEMORY))
            return new LRUCachingTileLoader<>(inner, 10);
          else if (cacheLevel.equals(CacheLevels.BYTE_ARRAY))
            return new LRUCachingTileLoader<>(inner, 100);
        }
        return new LRUCachingTileLoader<>(inner, 5);
      }

      @Override
      public <P, T> CachingDataLoader<P, T> createCachingDataLoader(
          final CacheLevel cacheLevel, final DataLoader<P, T> inner) {
        return new LRUCachingDataLoader<>(inner, 5);
      }
    });

If you need a different caching strategy from LRUCachingTileLoader, write a fully customized CachingTileLoader.
To manage the raw data (CacheLevel.ByteArray) in a special way, use the following:

Scala
Java
import com.here.platform.location.dataloader.core.TileLoader
import com.here.platform.location.dataloader.core.caching._

class DiskCachingLoader[Id, T](inner: DataLoader[Id, T]) extends CachingDataLoader[Id, T] {
  override def catalogHrn: HRN = inner.catalogHrn
  override def catalogVersion: Long = inner.catalogVersion
  override def layerName: String = inner.layerName
  override def partitioning: LayerPartitioning = inner.partitioning
  override def partitionIds: Iterator[Id] = inner.partitionIds

  override def get(id: Id): Option[T] =
    downloadIfAbsent(id).map(_.asInstanceOf[T])

  override def getAll(ids: Iterable[Id]): Iterable[(Id, Option[T])] =
    downloadIfAbsent(ids).map {
      case (id, value) => (id, value.map(_.asInstanceOf[T]))
    }

  override def stats: CacheStats =
    // tracking of cache misses, cache hits, etc.
    ???

  // Tries to fetch data from disk and if absent uses inner.get to download data
  private def downloadIfAbsent(id: Id): Option[Array[Byte]] =
    ???

  // Tries to fetch data from disk and if absent uses inner.getAll to download the missing data
  private def downloadIfAbsent(id: Iterable[Id]): Iterable[(Id, Option[Array[Byte]])] =
    ???
}

class DiskCachingPolicy extends CachingPolicy {
  override def createCachingTileLoader[T](cacheLevel: CacheLevel,
                                          inner: TileLoader[T]): CachingTileLoader[T] =
    cacheLevel match {
      case CacheLevel.ByteArray =>
        CachingDataLoader.asTileLoader(new DiskCachingLoader[TileId, T](inner))
      case _ => new LRUCachingTileLoader[T](inner, size = 5)
    }
  override def createCachingDataLoader[P, T](cacheLevel: CacheLevel,
                                             inner: DataLoader[P, T]): CachingDataLoader[P, T] =
    cacheLevel match {
      case CacheLevel.ByteArray =>
        new DiskCachingLoader(inner)
      case _ => new LRUCachingDataLoader(inner, size = 5)
    }
}

val cacheManager = CacheManager.withCachingPolicy(new DiskCachingPolicy)
import com.here.platform.location.dataloader.core.DataLoader;
import com.here.platform.location.dataloader.core.TileLoader;
import com.here.platform.location.dataloader.core.caching.*;
import com.here.platform.location.dataloader.core.caching.javadsl.CacheLevels;
import com.here.platform.location.dataloader.core.javadsl.AbstractTileLoaderDecorator;
import com.here.platform.location.inmemory.geospatial.TileId;
import scala.Option;
import scala.Tuple2;
import scala.collection.Iterable;
import scala.collection.JavaConverters;

import java.util.Collection;
import java.util.stream.Collectors;

class DiskCachingTileLoader<T> extends AbstractTileLoaderDecorator<T>
    implements CachingTileLoader<T> {

  public DiskCachingTileLoader(final TileLoader<T> inner) {
    super(inner);
  }

  @Override
  public CacheStats stats() {
    // tracking of cache misses, cache hits, etc.
    throw new UnsupportedOperationException();
  }

  @Override
  public Option<T> get(final long tileId) {
    final Option<byte[]> result = downloadIfAbsent(tileId);
    return parse(result);
  }

  @Override
  public Iterable<Tuple2<TileId, Option<T>>> getAll(final Iterable<TileId> tileIds) {
    final Collection<Tuple2<TileId, Option<byte[]>>> result =
        downloadIfAbsent(JavaConverters.asJavaCollectionConverter(tileIds).asJavaCollection());
    return JavaConverters.iterableAsScalaIterableConverter(
            result.stream()
                .map(tuple -> new Tuple2<>(tuple._1, parse(tuple._2)))
                .collect(Collectors.toList()))
        .asScala();
  }

  @SuppressWarnings("unchecked")
  private Option<T> parse(final Option<byte[]> result) {
    return (result.isDefined()) ? Option.apply((T) result.get()) : Option.empty();
  }

  // Tries to fetch data from disk and if absent uses inner.get to download data
  private Option<byte[]> downloadIfAbsent(final long tileId) {
    throw new UnsupportedOperationException();
  }

  // Tries to fetch data from disk and if absent uses inner.getAll to download the missing data
  private Collection<Tuple2<TileId, Option<byte[]>>> downloadIfAbsent(
      final Collection<TileId> tileIds) {
    throw new UnsupportedOperationException();
  }
}

class DiskCachingPolicy extends CachingPolicy {
  private static final long serialVersionUID = 1L;

  @Override
  public <T> CachingTileLoader<T> createCachingTileLoader(
      final CacheLevel cacheLevel, final TileLoader<T> inner) {
    return (cacheLevel.equals(CacheLevels.BYTE_ARRAY))
        ? new DiskCachingTileLoader<T>(inner)
        : new LRUCachingTileLoader<T>(inner, 5);
  }

  @Override
  public <P, T> CachingDataLoader<P, T> createCachingDataLoader(
      final CacheLevel cacheLevel, final DataLoader<P, T> inner) {
    return new LRUCachingDataLoader<>(inner, 5);
  }
}

final CacheManager cacheManager = CacheManager.withCachingPolicy(new DiskCachingPolicy());

results matching ""

    No results matching ""