JSONContainer.h

Go to the documentation of this file.

//
// Created by Nico Schäfer on 9/7/17.
//
 
#ifndef JODA_JSONCONTAINER_H
#define JODA_JSONCONTAINER_H
 
#include <joda/document/RapidJsonDocument.h>
#include <joda/indexing/QueryCache.h>
#include <joda/query/project/IProjector.h>
#include <joda/query/project/ISetProjector.h>
#include <limits.h>
 
#include <cstdio>
#include <functional>
#include <unordered_set>
#include <vector>
 
#include <joda/config/config.h>
#include <joda/document/RapidJsonDocument.h>
#include <joda/document/view/ViewLayer.h>
#include <joda/indexing/QueryCache.h>
#include <joda/query/project/IProjector.h>
#include <joda/query/project/ISetProjector.h>
#include <limits.h>
#include <functional>
#include <unordered_set>
#include "../../../../document/src/DocumentCostHandler.h"
#include "joda/misc/bloom_filter.hpp"
 
typedef std::vector<bool> DocIndex;
 
class JSONContainer {
 public:
  JSONContainer();
  explicit JSONContainer(size_t maxsize);
  JSONContainer(JSONContainer &other) = delete;
 
  virtual ~JSONContainer();
  void preparePurge();
 
  bool hasSpace(size_t size) const;
 
  bool hasMetaSpace(size_t size);
 
  unsigned long size() const;
 
  size_t getMaxSize() const;
 
  bool probContainsAttr(const std::string &attr) const;
 
  // Allocator
  RJMemoryPoolAlloc *getAlloc();
 
  // Documents
  void insertDoc(std::unique_ptr<RJDocument> &&doc,
                 std::unique_ptr<IOrigin> &&origin, size_t baseIndex = 0);
 
  void insertDoc(RapidJsonDocument &&doc, size_t baseIndex = 0);
 
  // Document Access
  std::unique_ptr<const DocIndex> checkDocuments(
      std::function<bool(RapidJsonDocument &, size_t)> &func);
 
  template <class retType>
  void forAll(std::function<retType(RapidJsonDocument &)> &func,
              std::vector<retType> &vec);
 
  template <class F>
  void forAll(F &f) {
    ScopedRef ref(this);
    for (auto &&doc : docs) {
      if (doc.isValid()) {
        f(doc);
      }
    }
    setLastUsed();
  }
 
  template <class F>
  void forAll(F f) {
    ScopedRef ref(this);
    for (auto &&doc : docs) {
      if (doc.isValid()) {
        f(doc);
      }
    }
    setLastUsed();
  }
 
  template <class F>
  void forAll(F f, const DocIndex &ids) {
    ScopedRef useCont(this, false);
    reparseSubset(ids);
    
    for (size_t i = 0; i < docs.size(); ++i) {
      if (!ids[i]) continue;
      auto &doc = docs[i];
      if (doc.isValid()) {
        f(doc);
      }
    }
    setLastUsed();
  }
 
  template <class F, class R>
  std::vector<R> forAllRet(F f, const DocIndex &ids) {
    auto ret = std::vector<R>();
    ret.reserve(ids.size());
 
    ScopedRef useCont(this, false);
    reparseSubset(ids);
 
    for (size_t i = 0; i < docs.size(); ++i) {
      if (!ids[i]) {
        ret.emplace_back();
        continue;
      };
      auto &doc = docs[i];
      if (doc.isValid()) {
        ret.emplace_back(f(doc,i));
      }
    }
    setLastUsed();
    return ret;
  }
 
  std::vector<std::unique_ptr<RJDocument>> projectDocuments(
      const DocIndex &ids,
      const std::vector<std::unique_ptr<joda::query::IProjector>> &proj,
      RJMemoryPoolAlloc &alloc,
      const std::vector<std::unique_ptr<joda::query::ISetProjector>> &setProj);
 
  std::unique_ptr<JSONContainer> createViewFromContainer(
      const DocIndex &ids,
      const std::vector<std::unique_ptr<joda::query::IProjector>> &proj,
      const std::vector<std::unique_ptr<joda::query::ISetProjector>> &setProj);
 
  DocumentCostHandler createTempViewDocs(
      const DocIndex &ids,
      const std::vector<std::unique_ptr<joda::query::IProjector>> &proj,
      const std::vector<std::unique_ptr<joda::query::ISetProjector>> &setProj);
 
  bool useViewBasedOnSample(
      const DocIndex &ids,
      const std::vector<std::unique_ptr<joda::query::IProjector>> &proj,
      const std::vector<std::unique_ptr<joda::query::ISetProjector>> &setProj);
  void finalize();
 
  void metaFinalize();
 
  void removeDocuments();
 
  void reparse();
 
  void setViews();
  void removeViews();
 
  void reparseSubset(unsigned long start = 0, unsigned long end = ULONG_MAX);
 
  void reparseSubset(const DocIndex &index);
 
  bool isReparsable();
 
  size_t estimatedSize() const;
 
  size_t parsedSize() const;
 
  /*
   * Stringify
   */
  std::vector<std::string> stringify(unsigned long start = 0,
                                     unsigned long end = ULONG_MAX);
 
  std::vector<std::unique_ptr<RJDocument>> getRaw(
      unsigned long start = 0, unsigned long end = ULONG_MAX);
  std::vector<std::unique_ptr<RJDocument>> getRaw(const DocIndex &ids);
  std::vector<std::unique_ptr<RJDocument>> getRaw(const DocIndex &ids,
                                                  RJMemoryPoolAlloc &alloc);
 
  template <class Handler>
  std::vector<bool> AcceptDocuments(Handler &handler, unsigned long start = 0,
                                    unsigned long end = ULONG_MAX) {
    std::vector<bool> ret;
    if (docs.empty()) return ret;
    ScopedRef useCont(this, false);
    reparseSubset(start, end);
    end = std::min(end, docs.size() - 1);
    if (start > end) return ret;
    for (unsigned long i = start; i <= end; ++i) {
      auto &doc = docs[i];
      if (doc.isValid()) {
        if (isView()) {
          auto &view = doc.getView();
          ret.push_back(view->Accept(handler));
        } else {
          ret.push_back(doc.getJson()->Accept(handler));
        }
      } else
        ret.push_back(false);
    }
    return ret;
  }
 
  void writeFile(const std::string &file, bool append);
 
  /*
   * Indices
   */
  const std::unique_ptr<QueryCache> &getCache() const;
  std::unique_ptr<const DocIndex> getAllIDs() const;
 
  void materializeAttributes(const std::vector<std::string> &atts);
  void materializeAttributesIfRequired(const std::vector<std::string> &atts);
  void materializeView();
 
  bool isView() const;
 
  unsigned long getLastUsed() const;
 
  bool isBaseContainer(const JSONContainer *cont) const;
 
  /*
   * Ref Counting
   */
  class ScopedRef {
   public:
    ScopedRef(JSONContainer *cont, bool parsed = true)
        : engaged_(true), cont_(cont) {
      cont_->useCont(parsed);
    }
 
    ~ScopedRef() {
      if (engaged_) {
        release();
      }
    }
 
    void release() {
      engaged_ = false;
      cont_->unUseCont();
    }
 
   private:
    bool engaged_;
    JSONContainer *cont_ = nullptr;
  };
 
  inline auto useContInScope(bool parse = true) { return ScopedRef(this, parse); }
 
 private:
  size_t lastParsedSize = 0;
  /*
   * Ref Counting
   */
  std::atomic<unsigned int> usage{0};
 
  inline void useCont(bool parse = true) {
    if (isView()) baseContainer->useCont(parse);
    auto prev = usage.fetch_add(1);
    if (parse) reparse();
    if (prev == 0) setViews();
  }
 
  inline void unUseCont() {
    if (isView()) baseContainer->unUseCont();
    auto prev = usage.fetch_sub(1);
    if (prev == 1 && !config::storeJson) removeDocuments();
  }
 
  // Indices
  DOC_ID minID = std::numeric_limits<DOC_ID>::max();
  DOC_ID maxID = 0;
  bloom_filter attr_bloom;
  bool bloomCalculated = false;
  bool viewsComputed = false;
  std::unique_ptr<QueryCache> cache;
 
  void calculateBloom();
  void recursiveBloomAttrSearch(const RJValue &obj,
                                const std::string &attr = "");
 
  bool serializeMissing();
  FILEID id = 0;
 
  std::vector<std::string> materializedAttributes;
  JSONContainer *baseContainer = nullptr;
  std::vector<JSONContainer *> subContainers;
  std::unique_ptr<ViewStructure> viewStruc;
 
  void addSubContainer(JSONContainer *cont);
  void removeSubContainer(JSONContainer *cont);
 
  void insertViewDoc(std::unique_ptr<RJDocument> &&doc, size_t baseIndex);
 
  // Size
  size_t maxSize;
  size_t theoreticalSize = 0;
 
  // Storage
  bool final = false;
  unsigned long lastUsed = 0;
  bool deleted = false;
 
  void setLastUsed();
 
 
  static bool compareDocContainer(const RapidJsonDocument &i,
                                  const RapidJsonDocument &j) {                              
    return i.getOrigin()->operator<(*j.getOrigin());
  }
 
  std::vector<RapidJsonDocument> docs;
  std::vector<size_t> baseIds;
 
  RJMemoryPoolPointer alloc;
};
 
template <class retType>
void JSONContainer::forAll(std::function<retType(RapidJsonDocument &)> &func,
                           std::vector<retType> &vec) {
  ScopedRef ref(this);
  for (auto &&doc : docs) {
    if (doc.isValid()) {
      vec.push_back(std::move(func(doc)));
    }
  }
  setLastUsed();
}
 
#endif  // JODA_JSONCONTAINER_H