Permalink
Please sign in to comment.
Browse files
Replace bash script with more flexible Python script (#3)
See additional discussion in #2
- Loading branch information...
Showing
with
132 additions
and 39 deletions.
- +0 −11 .docker/load.sh
- +0 −17 .docker/run.sh
- +2 −0 .gitignore
- +1 −4 Dockerfile
- +6 −7 README.md
- +16 −0 config.json
- 0 {.docker → }/configsets/core17/conf/lang/contractions_ca.txt
- 0 {.docker → }/configsets/core17/conf/lang/contractions_fr.txt
- 0 {.docker → }/configsets/core17/conf/lang/contractions_ga.txt
- 0 {.docker → }/configsets/core17/conf/lang/contractions_it.txt
- 0 {.docker → }/configsets/core17/conf/lang/hyphenations_ga.txt
- 0 {.docker → }/configsets/core17/conf/lang/stemdict_nl.txt
- 0 {.docker → }/configsets/core17/conf/lang/stoptags_ja.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_ar.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_bg.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_ca.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_cz.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_da.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_de.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_el.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_en.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_es.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_eu.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_fa.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_fi.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_fr.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_ga.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_gl.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_hi.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_hu.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_hy.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_id.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_it.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_ja.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_lv.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_nl.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_no.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_pt.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_ro.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_ru.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_sv.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_th.txt
- 0 {.docker → }/configsets/core17/conf/lang/stopwords_tr.txt
- 0 {.docker → }/configsets/core17/conf/lang/userdict_ja.txt
- 0 {.docker → }/configsets/core17/conf/managed-schema
- 0 {.docker → }/configsets/core17/conf/params.json
- 0 {.docker → }/configsets/core17/conf/protwords.txt
- 0 {.docker → }/configsets/core17/conf/solrconfig.xml
- 0 {.docker → }/configsets/core17/conf/stopwords.txt
- 0 {.docker → }/configsets/core17/conf/synonyms.txt
- 0 {.docker → }/configsets/mb11/conf/lang/contractions_ca.txt
- 0 {.docker → }/configsets/mb11/conf/lang/contractions_fr.txt
- 0 {.docker → }/configsets/mb11/conf/lang/contractions_ga.txt
- 0 {.docker → }/configsets/mb11/conf/lang/contractions_it.txt
- 0 {.docker → }/configsets/mb11/conf/lang/hyphenations_ga.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stemdict_nl.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stoptags_ja.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_ar.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_bg.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_ca.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_cz.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_da.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_de.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_el.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_en.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_es.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_eu.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_fa.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_fi.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_fr.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_ga.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_gl.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_hi.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_hu.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_hy.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_id.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_it.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_ja.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_lv.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_nl.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_no.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_pt.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_ro.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_ru.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_sv.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_th.txt
- 0 {.docker → }/configsets/mb11/conf/lang/stopwords_tr.txt
- 0 {.docker → }/configsets/mb11/conf/lang/userdict_ja.txt
- 0 {.docker → }/configsets/mb11/conf/managed-schema
- 0 {.docker → }/configsets/mb11/conf/params.json
- 0 {.docker → }/configsets/mb11/conf/protwords.txt
- 0 {.docker → }/configsets/mb11/conf/solrconfig.xml
- 0 {.docker → }/configsets/mb11/conf/stopwords.txt
- 0 {.docker → }/configsets/mb11/conf/synonyms.txt
- +107 −0 run.py
@@ -0,0 +1,2 @@ | |||
anserini.jar | |||
.idea/ |
@@ -0,0 +1,16 @@ | |||
{ | |||
"image_name": "solr-anserini", | |||
"index_mount": "/", | |||
"indexes": [ | |||
{ | |||
"name": "core17", | |||
"index_path": "../index/lucene-index.core17.pos+docvectors+rawdocs", | |||
"config_path": "configsets/core17" | |||
}, | |||
{ | |||
"name": "mb11", | |||
"index_path": "../index/lucene-index.mb11.pos+docvectors+rawdocs", | |||
"config_path": "configsets/mb11" | |||
} | |||
] | |||
} |
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,107 @@ | |||
import argparse | |||
import json | |||
import os | |||
import time | |||
|
|||
import docker | |||
from docker.errors import NotFound | |||
|
|||
|
|||
def run(config_path): | |||
# Load config file | |||
with open(config_path) as file: | |||
config = json.load(file) | |||
|
|||
# Docker API client | |||
client = docker.from_env() | |||
|
|||
# Remove any existing image | |||
remove_existing(client, config) | |||
|
|||
# Build the container | |||
build_container(client, config) | |||
|
|||
# Run the container | |||
container = run_container(client, config) | |||
|
|||
print("Sleeping for 10 seconds while Solr starts...") | |||
time.sleep(10) | |||
|
|||
for index in config["indexes"]: | |||
# Remove the lock file, if exists. | |||
container.exec_run("rm -f %s" % os.path.join(config["index_mount"], index["name"], "write.lock"), user='solr') | |||
|
|||
# Remove Solr generated index data. | |||
container.exec_run("rm -rf %s" % os.path.join("/opt/solr/server/solr/mycores", index["name"], "data/index"), | |||
user='solr') | |||
|
|||
# Create link to data volume | |||
container.exec_run("ln -s %s %s" % (os.path.join(config["index_mount"], index["name"]), | |||
os.path.join("/opt/solr/server/solr/mycores", index["name"], "data/index")), | |||
user='solr') | |||
|
|||
|
|||
# Remove any existing containers | |||
def remove_existing(client, config): | |||
try: | |||
container = client.containers.get(config["image_name"]) | |||
container.stop() | |||
container.remove() | |||
except NotFound: | |||
pass | |||
|
|||
|
|||
# Build the container | |||
def build_container(client, config): | |||
for line in client.api.build(path=".", tag=config["image_name"]): | |||
print(line) | |||
|
|||
|
|||
def get_volumes(config): | |||
volumes = {} | |||
|
|||
for index in config["indexes"]: | |||
# Path on host for index | |||
index_path_host = os.path.join(os.getcwd(), index["index_path"]) | |||
|
|||
# Path in container for index | |||
index_path_container = os.path.join(config["index_mount"], index["name"]) | |||
|
|||
# Path on host for configs | |||
config_path_host = os.path.join(os.getcwd(), index["config_path"]) | |||
|
|||
# Path in container for configs | |||
config_path_container = os.path.join("/opt/solr/server/solr/configsets", index["name"]) | |||
|
|||
# Add the binding for index paths | |||
volumes[index_path_host] = { | |||
"bind": index_path_container, | |||
"mode": "rw" | |||
} | |||
|
|||
# Add the binding for config paths | |||
volumes[config_path_host] = { | |||
"bind": config_path_container, | |||
"mode": "ro" | |||
} | |||
|
|||
return volumes | |||
|
|||
|
|||
def run_container(client, config): | |||
volumes = get_volumes(config) | |||
return client.containers.run(config["image_name"], | |||
detach=True, | |||
name=config["image_name"], | |||
ports={"8983": "8983"}, | |||
user="solr", | |||
volumes=volumes) | |||
|
|||
|
|||
if __name__ == '__main__': | |||
parser = argparse.ArgumentParser() | |||
parser.add_argument("--config", default="config.json", type=str, help="The config file location.") | |||
|
|||
args = parser.parse_args() | |||
|
|||
run(args.config) |
0 comments on commit
7321535