Welcome to Elasticmagic

Elasticmagic implements advanced type awareness DSL for Kotlin to construct Elasticsearch queries.

Warning

The library is in very alpha status. API may change significantly at any time. Use it on your own risk

Getting started

Setup

Add following dependencies in your build.gradle.kts script:

repositories {
    mavenCentral()
}

val elasticmagicVersion = "0.0.29"
val ktorVersion = "2.2.2"

dependencies {
    // Elasticmagic core api
    implementation("dev.evo.elasticmagic:elasticmagic:$elasticmagicVersion")
    // Json serialization using kotlinx.serialization
    implementation("dev.evo.elasticmagic:elasticmagic-serde-kotlinx-json:$elasticmagicVersion")
    // Transport that uses ktor http client
    implementation("dev.evo.elasticmagic:elasticmagic-transport-ktor:$elasticmagicVersion")

    implementation("io.ktor:ktor-client-cio:$ktorVersion")
}

Usage

First you need to describe a document (represents a mapping in terms of Elasticsearch):

package samples.started

import dev.evo.elasticmagic.doc.Document

object UserDoc : Document() {
    val id by int()
    val name by keyword()
    val groups by keyword()
    val about by text()
}

Now create ElasticsearchCluster object. It is an entry point for executing search queries:

package samples.started

import dev.evo.elasticmagic.ElasticsearchCluster
import dev.evo.elasticmagic.serde.kotlinx.JsonSerde
import dev.evo.elasticmagic.transport.ElasticsearchKtorTransport

const val DEFAULT_ELASTIC_URL = "http://localhost:9200"
const val DEFAULT_ELASTIC_USER = "elastic"

expect val esTransport: ElasticsearchKtorTransport
val cluster = ElasticsearchCluster(esTransport, serde = JsonSerde)
val userIndex = cluster["elasticmagic-samples_user"]

Any ElasticsearchCluster needs an ElasticsearchTransport. We will use the ElasticsearchKtorTransport that utilises Ktor http client.

Here are examples of creating transports for the cluster.

JVM:

package samples.started

import dev.evo.elasticmagic.transport.Auth
import dev.evo.elasticmagic.transport.ElasticsearchKtorTransport
import dev.evo.elasticmagic.transport.Request
import dev.evo.elasticmagic.transport.Response
import dev.evo.elasticmagic.transport.PlainRequest
import dev.evo.elasticmagic.transport.PlainResponse
import dev.evo.elasticmagic.transport.Tracker

import io.ktor.client.engine.cio.CIO

import java.security.cert.X509Certificate
import javax.net.ssl.X509TrustManager

import kotlin.time.Duration
import kotlin.getOrThrow

actual val esTransport = ElasticsearchKtorTransport(
    System.getenv("ELASTIC_URL") ?: DEFAULT_ELASTIC_URL,
    engine = CIO.create {
        https {
            trustManager = object: X509TrustManager {
                override fun checkClientTrusted(
                    chain: Array<out X509Certificate>?, authType: String?
                ) {}

                override fun checkServerTrusted(
                    chain: Array<out X509Certificate>?, authType: String?
                ) {}

                override fun getAcceptedIssuers(): Array<X509Certificate>? = null
            }
        }
    }
) {
    val elasticUser = System.getenv("ELASTIC_USER") ?: DEFAULT_ELASTIC_USER
    val elasticPassword = System.getenv("ELASTIC_PASSWORD")
    if (!elasticPassword.isNullOrEmpty()) {
        auth = Auth.Basic(elasticUser, elasticPassword)
    }

    if (System.getenv("ELASTICMAGIC_DEBUG") != null) {
        trackers = listOf {
            object : Tracker {
                override fun requiresTextContent(request: Request<*, *, *>) = true

                override suspend fun onRequest(request: PlainRequest) {
                    println(">>>")
                    println("${request.method} ${request.path.ifEmpty { "/" }}")
                    println(request.textContent)
                }

                override suspend fun onResponse(responseResult: Result<PlainResponse>, duration: Duration) {
                    responseResult
                        .onSuccess { response ->
                            println("<<< ${response.statusCode}: ${duration}")
                            response.headers.forEach { header ->
                                println("< ${header.key}: ${header.value}")
                            }
                            println(response.contentType)
                            println(response.content)
                        }
                        .onFailure { exception ->
                            println("!!! $exception")
                        }
                }
            }
        }
    }
}

Native:

package samples.started

import dev.evo.elasticmagic.serde.kotlinx.JsonSerde
import dev.evo.elasticmagic.transport.Auth
import dev.evo.elasticmagic.transport.ElasticsearchKtorTransport

import io.ktor.client.engine.curl.Curl

import kotlinx.cinterop.toKString

import platform.posix.getenv

actual val esTransport = ElasticsearchKtorTransport(
    getenv("ELASTIC_URL")?.toKString() ?: DEFAULT_ELASTIC_URL,
    engine = Curl.create {
        sslVerify = false
    }
) {
    val elasticUser = getenv("ELASTIC_USER")?.toKString() ?: DEFAULT_ELASTIC_USER
    val elasticPassword = getenv("ELASTIC_PASSWORD")?.toKString()
    if (!elasticPassword.isNullOrEmpty()) {
        auth = Auth.Basic(elasticUser, elasticPassword)
    }
}

Create our index if it does not exist or update the mapping otherwise:

package samples.started

import dev.evo.elasticmagic.Params

suspend fun ensureIndexExists() {
    if (!cluster.indexExists(userIndex.name)) {
        cluster.createIndex(
            userIndex.name,
            mapping = UserDoc,
            settings = Params(
                "index.number_of_replicas" to 0,
            ),
        )
    } else {
        cluster.updateMapping(userIndex.name, mapping = UserDoc)
    }
}

Describe document sources and index them:

package samples.started

import dev.evo.elasticmagic.Refresh
import dev.evo.elasticmagic.doc.DynDocSource
import dev.evo.elasticmagic.bulk.IndexAction
import dev.evo.elasticmagic.bulk.IdActionMeta
import dev.evo.elasticmagic.doc.list

suspend fun indexDocs() {
    val docs = listOf(
        DynDocSource {
            // Note that you can't write like following (it just won't compile):
            // it[UserDoc.id] = "0"
            // it[UserDoc.name] = 123
            // it[UserDoc.groups.list()] = "root"
            it[UserDoc.id] = 0
            it[UserDoc.name] = "root"
            it[UserDoc.groups.list()] = mutableListOf("root", "wheel")
            it[UserDoc.about] = "Super user"
        },
        DynDocSource {
            it[UserDoc.id] = 1
            it[UserDoc.name] = "daemon"
            it[UserDoc.groups.list()] = mutableListOf("daemon")
            it[UserDoc.about] = "Daemon user"
        },
        DynDocSource {
            it[UserDoc.id] = 65535
            it[UserDoc.name] = "nobody"
            it[UserDoc.groups.list()] = mutableListOf("nobody")
            it[UserDoc.about] = "Just nobody"
        },
        DynDocSource {
            it[UserDoc.id] = 65534
            it[UserDoc.name] = "noone"
            it[UserDoc.groups.list()] = mutableListOf("nobody")
            it[UserDoc.about] = "Another nobody"
        },
    )
    // Create index actions, make bulk request and refresh the index
    userIndex.bulk(
        docs.map { doc ->
            IndexAction(
                meta = IdActionMeta(id = doc[UserDoc.id].toString()),
                source = doc,
            )
        },
        refresh = Refresh.TRUE,
    )
}

And finally we can search our data:

package samples.started

import dev.evo.elasticmagic.SearchQuery
import dev.evo.elasticmagic.SearchQueryResult
import dev.evo.elasticmagic.aggs.TermsAgg
import dev.evo.elasticmagic.aggs.TermsAggResult
import dev.evo.elasticmagic.doc.DynDocSource
import dev.evo.elasticmagic.query.match

import kotlinx.coroutines.runBlocking

fun printUsers(result: SearchQueryResult<DynDocSource>) {
    println("Found: ${result.totalHits} users")
    for (hit in result.hits) {
        val user = hit.source!!
        println("  ${user[UserDoc.id]}: ${user[UserDoc.name]}")
    }
    println()
}

fun printGroupsAgg(aggResult: TermsAggResult<String>) {
    println("Groups aggregation")
    for (bucket in aggResult.buckets) {
        println("  ${bucket.key}: ${bucket.docCount}")
    }
}

fun main() = runBlocking {
    ensureIndexExists()
    indexDocs()

    // Find all users
    val sq = SearchQuery()
    printUsers(sq.search(userIndex))

    // Find nobody users
    sq.query(UserDoc.about.match("nobody"))
    printUsers(sq.search(userIndex))

    // Build an aggregation that counts users inside a group
    printGroupsAgg(
        SearchQuery()
            .aggs("groups" to TermsAgg(UserDoc.groups))
            .search(userIndex)
            .agg("groups")
    )
}

Run the sample

You can find fully working example inside samples

And run it with as JVM application (of cause you need Elasticsearch available at localhost:9200):

./gradlew :samples:run

or native:

./gradlew :samples:runDebugExecutableNative