[#2] Implement Nebulex.Adapter.Queryable behaviour

[#4] Support distributed adapters with Cachex

Author: cabol

.credo.exs

@@ -0,0 +1,31 @@
+%{
+  configs: [
+    %{
+      name: "default",
+      files: %{
+        included: ["lib/", "src/", "test/", "benchmarks/"],
+        excluded: [~r"/_build/", ~r"/deps/"]
+      },
+      color: true,
+      checks: [
+        # Design Checks
+        {Credo.Check.Design.AliasUsage, priority: :low},
+
+        # Deactivate due to they're not compatible with current Elixir version
+        {Credo.Check.Refactor.MapInto, false},
+        {Credo.Check.Warning.LazyLogging, false},
+
+        # Readability Checks
+        {Credo.Check.Readability.MaxLineLength, priority: :low, max_length: 100},
+
+        # Refactoring Opportunities
+        {Credo.Check.Refactor.LongQuoteBlocks, false},
+        {Credo.Check.Refactor.CyclomaticComplexity, max_complexity: 15},
+
+        # TODO and FIXME do not cause the build to fail
+        {Credo.Check.Design.TagTODO, exit_status: 0},
+        {Credo.Check.Design.TagFIXME, exit_status: 0}
+      ]
+    }
+  ]
+}

.github/workflows/ci.yml

@@ -16,6 +16,9 @@ jobs:
     strategy:
       matrix:
         include:
+          - elixir: 1.11.x
+            otp: 23.x
+            coverage: true
           - elixir: 1.10.x
             otp: 23.x
           - elixir: 1.10.x
@@ -26,17 +29,19 @@ jobs:
     env:
       GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
       MIX_ENV: test
-      NBX_TEST: true
+      NEBULEX_PATH: nebulex
 
     steps:
       - uses: actions/checkout@v2
 
-      - uses: actions/setup-elixir@v1
+      - name: Install OTP and Elixir
+        uses: actions/setup-elixir@v1
         with:
           otp-version: '${{ matrix.otp }}'
           elixir-version: '${{ matrix.elixir }}'
 
-      - uses: actions/cache@v1
+      - name: Cache deps
+        uses: actions/cache@v1
         with:
           path: deps
           key: >-
@@ -45,7 +50,8 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-
 
-      - uses: actions/cache@v1
+      - name: Cache _build
+        uses: actions/cache@v1
         with:
           path: _build
           key: >-
@@ -59,6 +65,7 @@ jobs:
           mix local.hex --force
           mix local.rebar --force
           mix deps.get
+          git clone --depth 1 --branch master https://github.com/cabol/nebulex.git
 
       - name: Run style and code consistency checks
         run: |
@@ -67,11 +74,19 @@ jobs:
           mix credo --strict
 
       - name: Run tests
+        run: |
+          epmd -daemon
+          mix test --trace
+        if: ${{!matrix.coverage}}
+
+      - name: Run tests with coverage
         run: |
           epmd -daemon
           mix coveralls.github
+        if: ${{matrix.coverage}}
 
-      - uses: actions/cache@v1
+      - name: Cache PLT
+        uses: actions/cache@v1
         with:
           path: priv/plts
           key: '${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-plt-v1'

.gitignore

@@ -36,4 +36,4 @@ erl_crash.dump
 .vs*
 /priv
 .sobelow*
-mix.lock
+/nebulex

lib/nebulex_cachex_adapter.ex

@@ -5,9 +5,12 @@ defmodule NebulexCachexAdapter do
 
   # Provide Cache Implementation
   @behaviour Nebulex.Adapter
+  @behaviour Nebulex.Adapter.Queryable
 
   import Nebulex.Helpers
 
+  alias Cachex.Query
+
   @compile {:inline, to_ttl: 1}
 
   ## Adapter
@@ -57,6 +60,8 @@ defmodule NebulexCachexAdapter do
   end
 
   def put(%{name: name}, key, value, ttl, :put_new, _opts) do
+    # FIXME: This is a workaround since Cachex does not support a direct action
+    #        for put_new. Fix it if a better solution comes up.
     if Cachex.get!(name, key) do
       false
     else
@@ -76,6 +81,8 @@ defmodule NebulexCachexAdapter do
   def put_all(%{name: name}, entries, ttl, :put_new, _opts) when is_list(entries) do
     {keys, _} = Enum.unzip(entries)
 
+    # FIXME: This is a workaround since Cachex does not support a direct action
+    #        for put_new. Fix it if a better solution comes up.
     Cachex.transaction!(name, keys, fn worker ->
       if Enum.any?(keys, &(worker |> Cachex.exists?(&1) |> elem(1))) do
         false
@@ -105,12 +112,15 @@ defmodule NebulexCachexAdapter do
   @impl true
   def ttl(%{name: name}, key) do
     cond do
+      # Key does exist and has a TTL associated with it
       ttl = Cachex.ttl!(name, key) ->
         ttl
 
+      # Key does exist and hasn't a TTL associated with it
       Cachex.get!(name, key) ->
         :infinity
 
+      # Key does not exist
       true ->
         nil
     end
@@ -132,6 +142,8 @@ defmodule NebulexCachexAdapter do
   end
 
   def incr(%{name: name}, key, incr, ttl, opts) do
+    # FIXME: This is a workaround since Cachex does not support `:ttl` here.
+    #        Fix it if a better solution comes up.
     Cachex.transaction!(name, [key], fn worker ->
       counter = Cachex.incr!(worker, key, incr, initial: opts[:default] || 0)
       if ttl = to_ttl(ttl), do: Cachex.expire!(worker, key, ttl)
@@ -151,6 +163,27 @@ defmodule NebulexCachexAdapter do
     size
   end
 
+  ## Queryable
+
+  @impl true
+  def all(adapter_meta, query, opts) do
+    adapter_meta
+    |> stream(query, opts)
+    |> Enum.to_list()
+  end
+
+  @impl true
+  def stream(adapter_meta, nil, opts) do
+    stream(adapter_meta, Query.create(true, :key), opts)
+  end
+
+  def stream(%{name: name}, query, opts) do
+    Cachex.stream!(name, query, batch_size: opts[:page_size] || 100)
+  rescue
+    e in Cachex.ExecutionError ->
+      reraise Nebulex.QueryError, [message: e.message, query: query], __STACKTRACE__
+  end
+
   ## Private Functions
 
   defp to_ttl(:infinity), do: nil

mix.exs

@@ -52,6 +52,7 @@ defmodule NebulexCachexAdapter.MixProject do
       {:credo, "~> 1.5", only: [:dev, :test], runtime: false},
       {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false},
       {:sobelow, "~> 0.10", only: [:dev, :test], runtime: false},
+      {:stream_data, "~> 0.5", only: [:dev, :test]},
 
       # Benchmark Test
       {:benchee, "~> 1.0", only: :test},
@@ -63,12 +64,8 @@ defmodule NebulexCachexAdapter.MixProject do
   end
 
   defp nebulex_dep do
-    if System.get_env("NBX_TEST") do
-      # This is because the adapter tests need some support modules and shared
-      # tests from nebulex dependency, and the hex dependency doesn't include
-      # the test folder. Hence, to run the tests it is necessary to fetch
-      # nebulex dependency directly from GH.
-      {:nebulex, github: "cabol/nebulex", branch: "master"}
+    if path = System.get_env("NEBULEX_PATH") do
+      {:nebulex, "~> 2.0.0-rc.1", path: path}
     else
       {:nebulex, "~> 2.0.0-rc.1"}
     end

mix.lock

@@ -0,0 +1,33 @@
+%{
+  "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm", "3ad58ae787e9c7c94dd7ceda3b587ec2c64604563e049b2a0e8baafae832addb"},
+  "benchee_html": {:hex, :benchee_html, "1.0.0", "5b4d24effebd060f466fb460ec06576e7b34a00fc26b234fe4f12c4f05c95947", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:benchee_json, "~> 1.0", [hex: :benchee_json, repo: "hexpm", optional: false]}], "hexpm", "5280af9aac432ff5ca4216d03e8a93f32209510e925b60e7f27c33796f69e699"},
+  "benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"},
+  "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"},
+  "cachex": {:hex, :cachex, "3.3.0", "6f2ebb8f27491fe39121bd207c78badc499214d76c695658b19d6079beeca5c2", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "d90e5ee1dde14cef33f6b187af4335b88748b72b30c038969176cd4e6ccc31a1"},
+  "certifi": {:hex, :certifi, "2.5.3", "70bdd7e7188c804f3a30ee0e7c99655bc35d8ac41c23e12325f36ab449b70651", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "ed516acb3929b101208a9d700062d520f3953da3b6b918d866106ffa980e1c10"},
+  "credo": {:hex, :credo, "1.5.4", "9914180105b438e378e94a844ec3a5088ae5875626fc945b7c1462b41afc3198", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cf51af45eadc0a3f39ba13b56fdac415c91b34f7b7533a13dc13550277141bc4"},
+  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
+  "dialyxir": {:hex, :dialyxir, "1.0.0", "6a1fa629f7881a9f5aaf3a78f094b2a51a0357c843871b8bc98824e7342d00a5", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "aeb06588145fac14ca08d8061a142d52753dbc2cf7f0d00fc1013f53f8654654"},
+  "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"},
+  "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
+  "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},
+  "ex_doc": {:hex, :ex_doc, "0.23.0", "a069bc9b0bf8efe323ecde8c0d62afc13d308b1fa3d228b65bca5cf8703a529d", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f5e2c4702468b2fd11b10d39416ddadd2fcdd173ba2a0285ebd92c39827a5a16"},
+  "excoveralls": {:hex, :excoveralls, "0.13.4", "7b0baee01fe150ef81153e6ffc0fc68214737f54570dc257b3ca4da8e419b812", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "faae00b3eee35cdf0342c10b669a7c91f942728217d2a7c7f644b24d391e6190"},
+  "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
+  "hackney": {:hex, :hackney, "1.17.0", "717ea195fd2f898d9fe9f1ce0afcc2621a41ecfe137fae57e7fe6e9484b9aa99", [:rebar3], [{:certifi, "~>2.5", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "64c22225f1ea8855f584720c0e5b3cd14095703af1c9fbc845ba042811dc671c"},
+  "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
+  "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"},
+  "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
+  "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.15.0", "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "75ffa34ab1056b7e24844c90bfc62aaf6f3a37a15faa76b07bc5eba27e4a8b4a"},
+  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
+  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"},
+  "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
+  "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"},
+  "sobelow": {:hex, :sobelow, "0.10.6", "ac9ebd742035b119eb00a4b1416098b88a4d54d2f262a42602e1e9e3ed4c2afd", [:mix], [], "hexpm", "06e426b8dc0bc80ab1333ce89b904c720474824825b9ceb3dc424eb0f731b6e9"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
+  "stream_data": {:hex, :stream_data, "0.5.0", "b27641e58941685c75b353577dc602c9d2c12292dd84babf506c2033cd97893e", [:mix], [], "hexpm", "012bd2eec069ada4db3411f9115ccafa38540a3c78c4c0349f151fc761b9e271"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
+  "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"},
+}

test/nebulex_cachex_adapter/local_test.exs

@@ -2,18 +2,9 @@ defmodule NebulexCachexAdapter.LocalTest do
   use ExUnit.Case, async: true
   use NebulexCachexAdapter.CacheTest
 
-  alias NebulexCachexAdapter.TestCache.Local, as: Cache
-
-  setup do
-    {:ok, pid} = Cache.start_link()
-    _ = Cache.flush()
-    :ok
+  import Nebulex.CacheCase
 
-    on_exit(fn ->
-      :ok = Process.sleep(100)
-      if Process.alive?(pid), do: Cache.stop(pid)
-    end)
+  alias NebulexCachexAdapter.TestCache.Local, as: Cache
 
-    {:ok, cache: Cache, name: Cache}
-  end
+  setup_with_dynamic_cache(Cache, :local_with_cachex)
 end

test/nebulex_cachex_adapter/multilevel_test.exs

@@ -0,0 +1,35 @@
+defmodule NebulexCachexAdapter.MultilevelTest do
+  use ExUnit.Case, async: true
+  use Nebulex.NodeCase
+  use Nebulex.MultilevelTest
+  # use Nebulex.Cache.QueryableTest
+  # use Nebulex.Cache.TransactionTest
+
+  import Nebulex.CacheCase
+
+  alias Nebulex.Time
+  alias NebulexCachexAdapter.TestCache.Multilevel
+  alias NebulexCachexAdapter.TestCache.Multilevel.{L1, L2, L3}
+
+  @gc_interval Time.expiry_time(1, :hour)
+
+  @levels [
+    {
+      L1,
+      name: :multilevel_inclusive_l1, gc_interval: @gc_interval
+    },
+    {
+      L2,
+      name: :multilevel_inclusive_l2, primary: [gc_interval: @gc_interval]
+    },
+    {
+      L3,
+      name: :multilevel_inclusive_l3, primary: [gc_interval: @gc_interval]
+    }
+  ]
+
+  setup_with_dynamic_cache(Multilevel, :multilevel_inclusive,
+    model: :inclusive,
+    levels: @levels
+  )
+end

test/nebulex_cachex_adapter/partitioned_test.exs

@@ -0,0 +1,41 @@
+defmodule NebulexCachexAdapter.PartitionedTest do
+  use Nebulex.NodeCase
+  use NebulexCachexAdapter.CacheTest
+
+  alias NebulexCachexAdapter.TestCache.Partitioned
+
+  @primary :"primary@127.0.0.1"
+
+  setup do
+    cluster =
+      :lists.usort([
+        @primary
+        | Application.get_env(:nebulex_cachex_adapter, :nodes, [])
+      ])
+
+    node_pid_list =
+      start_caches(
+        [node() | Node.list()],
+        [{Partitioned, []}]
+      )
+
+    on_exit(fn ->
+      :ok = Process.sleep(100)
+      stop_caches(node_pid_list)
+    end)
+
+    {:ok, cache: Partitioned, name: Partitioned, cluster: cluster}
+  end
+
+  describe "partitioned cache" do
+    test "get_and_update" do
+      assert Partitioned.get_and_update(1, &Partitioned.get_and_update_fun/1) == {nil, 1}
+      assert Partitioned.get_and_update(1, &Partitioned.get_and_update_fun/1) == {1, 2}
+      assert Partitioned.get_and_update(1, &Partitioned.get_and_update_fun/1) == {2, 4}
+
+      assert_raise ArgumentError, fn ->
+        Partitioned.get_and_update(1, &Partitioned.get_and_update_bad_fun/1)
+      end
+    end
+  end
+end

test/nebulex_cachex_adapter/replicated_test.exs

@@ -0,0 +1,17 @@
+defmodule NebulexCachexAdapter.ReplicatedTest do
+  use Nebulex.NodeCase
+  use NebulexCachexAdapter.CacheTest
+
+  alias NebulexCachexAdapter.TestCache.Replicated
+
+  setup do
+    node_pid_list = start_caches([node() | Node.list()], [{Replicated, []}])
+
+    on_exit(fn ->
+      :ok = Process.sleep(100)
+      stop_caches(node_pid_list)
+    end)
+
+    {:ok, cache: Replicated, name: Replicated}
+  end
+end