mirror of
https://github.com/PhoenixTwoFive/karaoqueue.git
synced 2025-05-19 19:11:49 +02:00
Begin migrating backend to MongoDB
This commit is contained in:
parent
8687408d9c
commit
d3aea64880
@ -2,12 +2,14 @@
|
||||
|
||||
import sqlite3
|
||||
import pandas
|
||||
import pymongo
|
||||
from bson.regex import Regex
|
||||
from io import StringIO
|
||||
|
||||
song_table = "songs"
|
||||
entry_table = "entries"
|
||||
index_label = "Id"
|
||||
done_table = "done_songs"
|
||||
db_name = "karaoqueue"
|
||||
song_collection_name = "songs"
|
||||
entry_collection_name = "entries"
|
||||
playback_collection_name = "played"
|
||||
|
||||
|
||||
def dict_factory(cursor, row):
|
||||
@ -16,102 +18,99 @@ def dict_factory(cursor, row):
|
||||
d[col[0]] = row[idx]
|
||||
return d
|
||||
|
||||
def open_db():
|
||||
conn = sqlite3.connect("data/test.db")
|
||||
conn.execute('PRAGMA encoding = "UTF-8";')
|
||||
return conn
|
||||
def open_db_client():
|
||||
mongoClient = pymongo.MongoClient("mongodb://localhost:27017")
|
||||
return mongoClient
|
||||
|
||||
def import_songs(song_csv):
|
||||
print("Start importing Songs...")
|
||||
df = pandas.read_csv(StringIO(song_csv), sep=';')
|
||||
conn = open_db()
|
||||
cur = conn.cursor()
|
||||
df.to_sql(song_table, conn, if_exists='replace',
|
||||
index=False)
|
||||
cur.execute("SELECT Count(Id) FROM songs")
|
||||
num_songs = cur.fetchone()[0]
|
||||
conn.close()
|
||||
client = open_db_client()
|
||||
db = client[db_name]
|
||||
if not song_collection_name in db.list_collection_names():
|
||||
songsCollection = db[song_collection_name]
|
||||
songsCollection.create_index("karafun_id", unique=True)
|
||||
songsCollection.create_index([("title","text"),("artist","text")])
|
||||
else:
|
||||
songsCollection = db[song_collection_name]
|
||||
|
||||
def f(x): return (x.split(","))
|
||||
|
||||
df = pandas.read_csv(StringIO(song_csv), sep=';',
|
||||
engine='python', parse_dates=["Date Added"])
|
||||
df.Styles = df.Styles.apply(f, convert_dtype=True)
|
||||
df.Languages = df.Languages.apply(f, convert_dtype=True)
|
||||
df.Duo = df.Duo.astype('bool')
|
||||
df.Explicit = df.Explicit.astype('bool')
|
||||
df.columns = map(str.lower, df.columns)
|
||||
df.rename(columns={'id': 'karafun_id'}, inplace=True)
|
||||
num_songs = df.shape[0]
|
||||
song_dict = df.to_dict('records')
|
||||
try:
|
||||
songsCollection.insert_many(song_dict)
|
||||
except pymongo.errors.BulkWriteError as bwe:
|
||||
return(bwe.details)
|
||||
finally:
|
||||
client.close()
|
||||
print("Imported songs ({} in Database)".format(num_songs))
|
||||
return("Imported songs ({} in Database)".format(num_songs))
|
||||
|
||||
def create_entry_table():
|
||||
conn = open_db()
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS '+entry_table +
|
||||
' (ID INTEGER PRIMARY KEY NOT NULL, Song_Id INTEGER NOT NULL, Name VARCHAR(255))')
|
||||
conn.close()
|
||||
|
||||
|
||||
def create_done_song_table():
|
||||
conn = open_db()
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS '+done_table +
|
||||
' (Song_Id INTEGER PRIMARY KEY NOT NULL, Plays INTEGER)')
|
||||
conn.close()
|
||||
|
||||
def create_song_table():
|
||||
conn = open_db()
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS \""+song_table+"""\" (
|
||||
"Id" INTEGER,
|
||||
"Title" TEXT,
|
||||
"Artist" TEXT,
|
||||
"Year" INTEGER,
|
||||
"Duo" INTEGER,
|
||||
"Explicit" INTEGER,
|
||||
"Date Added" TEXT,
|
||||
"Styles" TEXT,
|
||||
"Languages" TEXT
|
||||
)""")
|
||||
conn.close()
|
||||
|
||||
def create_list_view():
|
||||
conn = open_db()
|
||||
conn.execute("""CREATE VIEW IF NOT EXISTS [Liste] AS
|
||||
SELECT Name, Title, Artist, entries.Id, songs.Id
|
||||
FROM entries, songs
|
||||
WHERE entries.Song_Id=songs.Id""")
|
||||
conn.close()
|
||||
|
||||
|
||||
def create_done_song_view():
|
||||
conn = open_db()
|
||||
conn.execute("""CREATE VIEW IF NOT EXISTS [Abspielliste] AS
|
||||
SELECT Artist || \" - \" || Title AS Song, Plays AS Wiedergaben
|
||||
FROM songs, done_songs
|
||||
WHERE done_songs.Song_Id=songs.Id""")
|
||||
conn.close()
|
||||
return("Imported songs ({} in Database)".format(num_songs))
|
||||
|
||||
def get_list():
|
||||
conn = open_db()
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT * FROM Liste")
|
||||
return cur.fetchall()
|
||||
client = open_db_client()
|
||||
db = client[db_name]
|
||||
collection = db[entry_collection_name]
|
||||
|
||||
query = {}
|
||||
cursor = collection.find()
|
||||
|
||||
result = {}
|
||||
|
||||
try:
|
||||
for doc in cursor:
|
||||
result += doc
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
def get_played_list():
|
||||
conn = open_db()
|
||||
conn = open_db_client()
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT * FROM Abspielliste")
|
||||
return cur.fetchall()
|
||||
|
||||
def get_song_list():
|
||||
conn =open_db()
|
||||
conn =open_db_client()
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT Artist || \" - \" || Title AS Song, Id FROM songs;")
|
||||
return cur.fetchall()
|
||||
|
||||
def get_song_completions(input_string):
|
||||
conn = open_db()
|
||||
conn.row_factory = dict_factory
|
||||
cur = conn.cursor()
|
||||
# Don't look, it burns...
|
||||
prepared_string = "%{0}%".format(input_string).upper() # "Test" -> "%TEST%"
|
||||
print(prepared_string)
|
||||
cur.execute(
|
||||
"SELECT * FROM songs WHERE REPLACE(REPLACE(REPLACE(REPLACE(UPPER( Title ),'ö','Ö'),'ü','Ü'),'ä','Ä'),'ß','ẞ') LIKE (?) LIMIT 20;", (prepared_string,))
|
||||
return cur.fetchall()
|
||||
client = open_db_client()
|
||||
db = client[db_name]
|
||||
collection = db[song_collection_name]
|
||||
|
||||
cursor = collection.find({'$text': {'$search': input_string}}, {'_txtscr': {'$meta': 'textScore'}}, limit=30).sort([('_txtscr', {'$meta': 'textScore'})])
|
||||
|
||||
result = []
|
||||
|
||||
try:
|
||||
for doc in cursor:
|
||||
tmpdoc = doc
|
||||
tmpdoc["_id"] = str(tmpdoc["_id"])
|
||||
result.append(doc)
|
||||
finally:
|
||||
client.close()
|
||||
# conn = open_db_client()
|
||||
# conn.row_factory = dict_factory
|
||||
# cur = conn.cursor()
|
||||
# # Don't look, it burns...
|
||||
# prepared_string = "%{0}%".format(input_string).upper() # "Test" -> "%TEST%"
|
||||
# print(prepared_string)
|
||||
# cur.execute(
|
||||
# "SELECT * FROM songs WHERE REPLACE(REPLACE(REPLACE(REPLACE(UPPER( Title ),'ö','Ö'),'ü','Ü'),'ä','Ä'),'ß','ẞ') LIKE (?) LIMIT 20;", (prepared_string,))
|
||||
print(result)
|
||||
return result
|
||||
|
||||
def add_entry(name,song_id):
|
||||
conn = open_db()
|
||||
conn = open_db_client()
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"INSERT INTO entries (Song_Id,Name) VALUES(?,?);", (song_id,name))
|
||||
@ -120,7 +119,7 @@ def add_entry(name,song_id):
|
||||
return
|
||||
|
||||
def add_sung_song(entry_id):
|
||||
conn = open_db()
|
||||
conn = open_db_client()
|
||||
cur = conn.cursor()
|
||||
cur.execute("""SELECT Song_Id FROM entries WHERE Id=?""",(entry_id,))
|
||||
song_id = cur.fetchone()[0]
|
||||
@ -136,7 +135,7 @@ def add_sung_song(entry_id):
|
||||
return True
|
||||
|
||||
def clear_played_songs():
|
||||
conn = open_db()
|
||||
conn = open_db_client()
|
||||
cur = conn.cursor()
|
||||
cur.execute("DELETE FROM done_songs")
|
||||
conn.commit()
|
||||
@ -144,7 +143,7 @@ def clear_played_songs():
|
||||
return True
|
||||
|
||||
def delete_entry(id):
|
||||
conn = open_db()
|
||||
conn = open_db_client()
|
||||
cur = conn.cursor()
|
||||
cur.execute("DELETE FROM entries WHERE id=?",(id,))
|
||||
conn.commit()
|
||||
@ -157,7 +156,7 @@ def delete_entries(ids):
|
||||
for x in ids:
|
||||
idlist.append( (x,) )
|
||||
try:
|
||||
conn = open_db()
|
||||
conn = open_db_client()
|
||||
cur = conn.cursor()
|
||||
cur.executemany("DELETE FROM entries WHERE id=?", idlist)
|
||||
conn.commit()
|
||||
@ -167,7 +166,7 @@ def delete_entries(ids):
|
||||
return -1
|
||||
|
||||
def delete_all_entries():
|
||||
conn = open_db()
|
||||
conn = open_db_client()
|
||||
cur = conn.cursor()
|
||||
cur.execute("DELETE FROM entries")
|
||||
conn.commit()
|
||||
|
||||
@ -2,6 +2,8 @@ import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
import os
|
||||
import datetime
|
||||
import bson.objectid
|
||||
|
||||
data_directory = "data"
|
||||
config_file = data_directory+"/config.json"
|
||||
@ -36,4 +38,11 @@ def setup_config(app):
|
||||
json.dump(config, handle, indent=4, sort_keys=True)
|
||||
print("Wrote new config")
|
||||
app.config['BASIC_AUTH_USERNAME'] = config['username']
|
||||
app.config['BASIC_AUTH_PASSWORD'] = config['password']
|
||||
app.config['BASIC_AUTH_PASSWORD'] = config['password']
|
||||
|
||||
|
||||
def serialization_helper(obj):
|
||||
if isinstance(obj, datetime.datetime):
|
||||
return obj.isoformat()
|
||||
elif isinstance(obj, bson.objectid.ObjectId):
|
||||
return str(obj)
|
||||
@ -5,7 +5,9 @@ import database
|
||||
import data_adapters
|
||||
import os, errno
|
||||
import json
|
||||
|
||||
from flask_basicauth import BasicAuth
|
||||
from pprint import pprint
|
||||
app = Flask(__name__, static_url_path='/static')
|
||||
|
||||
CORS(app)
|
||||
@ -55,7 +57,7 @@ def songs():
|
||||
@app.route("/api/songs/update")
|
||||
@basic_auth.required
|
||||
def update_songs():
|
||||
database.delete_all_entries()
|
||||
# database.delete_all_entries()
|
||||
status = database.import_songs(helpers.get_songs(helpers.get_catalog_url()))
|
||||
print(status)
|
||||
return Response('{"status": "%s" }' % status, mimetype='text/json')
|
||||
@ -67,7 +69,7 @@ def get_song_completions(input_string=""):
|
||||
if input_string!="":
|
||||
print(input_string)
|
||||
list = database.get_song_completions(input_string=input_string)
|
||||
return Response(json.dumps(list).encode('utf-8'), mimetype='application/json')
|
||||
return Response(json.dumps(list, default=helpers.serialization_helper).encode('utf-8'), mimetype='application/json')
|
||||
# return Response(json.dumps(list, ensure_ascii=False).encode('utf-8'), mimetype='text/json')
|
||||
|
||||
else:
|
||||
@ -146,11 +148,6 @@ def admin():
|
||||
@app.before_first_request
|
||||
def activate_job():
|
||||
helpers.create_data_directory()
|
||||
database.create_entry_table()
|
||||
database.create_song_table()
|
||||
database.create_done_song_table()
|
||||
database.create_list_view()
|
||||
database.create_done_song_view()
|
||||
helpers.setup_config(app)
|
||||
|
||||
|
||||
|
||||
@ -18,73 +18,11 @@ table td:nth-child(2) {
|
||||
aktualisieren</button>
|
||||
<input id="entryToggle" type="checkbox" class="topbutton" data-toggle="toggle" data-on="Eintragen erlaubt" data-off="Eintragen deaktiviert" data-onstyle="success" data-offstyle="danger">
|
||||
</div>
|
||||
<table class="table entries"
|
||||
id="entrytable"
|
||||
data-toggle="table"
|
||||
data-search="true"
|
||||
data-show-columns="true"
|
||||
data-show-toggle="true"
|
||||
data-multiple-select-row="true"
|
||||
data-click-to-select="true"
|
||||
data-toolbar="#toolbar"
|
||||
data-pagination="true"
|
||||
data-show-extended-pagination="true"
|
||||
data-classes="table table-hover"
|
||||
data-url="/api/queue"
|
||||
data-show-refresh="true"
|
||||
data-auto-refresh="true"
|
||||
data-auto-refresh-interval="10">
|
||||
<thead>
|
||||
<tr>
|
||||
<th data-field="state" data-checkbox="true"></th>
|
||||
<th scope="col" data-field="Name">Name</th>
|
||||
<th scope="col" data-field="Title">Song</th>
|
||||
<th scope="col" data-field="Artist">Künstler</th>
|
||||
<th scope="col" data-formatter="TableActions">Aktionen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
</table>
|
||||
<a name="end"></a>
|
||||
</div>
|
||||
{% endblock %}
|
||||
{% block extrajs %}
|
||||
<script>
|
||||
$(function () {
|
||||
$('#entryToggle').change(function() {
|
||||
$.ajax({url: "/api/entries/accept/"+($('#entryToggle').is(":checked") ? "1" : "0"), complete: setTimeout(refreshEntryToggle, 1000)});
|
||||
})
|
||||
refreshEntryToggle()
|
||||
$("#entrytable").bootstrapTable().on('load-success.bs.table', function() {
|
||||
$('[data-toggle="tooltip"]').tooltip()
|
||||
})
|
||||
})
|
||||
function confirmDeleteEntry(name, entry_id) {
|
||||
bootbox.confirm("Wirklich den Eintrag von "+name+" löschen?", function(result){
|
||||
if (result) {
|
||||
deleteEntry(entry_id)
|
||||
}
|
||||
})
|
||||
}
|
||||
function confirmDeleteSelectedEntries() {
|
||||
bootbox.confirm({
|
||||
message: "Wirklich gewählte Eintragungen löschen?",
|
||||
buttons: {
|
||||
confirm: {
|
||||
label: 'Ja',
|
||||
className: 'btn btn-danger'
|
||||
},
|
||||
cancel: {
|
||||
label: 'Nein',
|
||||
className: 'btn btn-secondary'
|
||||
}
|
||||
},
|
||||
callback: function(result){
|
||||
if (result) {
|
||||
DeleteSelectedEntries(getIdSelections())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
function confirmUpdateSongDatabase() {
|
||||
bootbox.confirm({
|
||||
message: "Wirklich die Song-Datenbank aktualisieren?<br>Dies lädt die Aktuelle Song-Liste von <a href='https://www.karafun.de/karaoke-song-list.html'>KaraFun</a> herunter, <b>und wird alle Eintragungen löschen!</b>",
|
||||
@ -110,58 +48,6 @@ table td:nth-child(2) {
|
||||
}
|
||||
})
|
||||
}
|
||||
function refreshEntryToggle() {
|
||||
$.getJSON("/api/entries/accept", (data) => {
|
||||
if (data["value"]!=$('#entryToggle').is(":checked")) {
|
||||
if(data["value"]==1) {
|
||||
$('#entryToggle').bootstrapToggle('on')
|
||||
}
|
||||
else {
|
||||
$('#entryToggle').bootstrapToggle('off')
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
function deleteEntry(entry_id) {
|
||||
$.ajax({
|
||||
type: 'GET',
|
||||
url: '/api/entries/delete/'+entry_id,
|
||||
contentType: "application/json",
|
||||
dataType: 'json',
|
||||
async: false
|
||||
});
|
||||
$("#entrytable").bootstrapTable('refresh')
|
||||
|
||||
}
|
||||
function markEntryAsSung(entry_id) {
|
||||
$.ajax({
|
||||
type: 'GET',
|
||||
url: '/api/entries/mark_sung/'+entry_id,
|
||||
contentType: "application/json",
|
||||
dataType: 'json',
|
||||
async: false
|
||||
});
|
||||
$("#entrytable").bootstrapTable('refresh')
|
||||
|
||||
}
|
||||
function DeleteSelectedEntries(ids) {
|
||||
$.ajax({
|
||||
type: 'POST',
|
||||
url: '/api/entries/delete',
|
||||
data: JSON.stringify(ids), // or JSON.stringify ({name: 'jonas'}),
|
||||
error: function() {
|
||||
bootbox.alert({
|
||||
message: "Fehler beim Löschen der Eintragungen.",
|
||||
})
|
||||
},
|
||||
success: function() {
|
||||
$("#entrytable").bootstrapTable('refresh')
|
||||
|
||||
},
|
||||
contentType: "application/json",
|
||||
dataType: 'json'
|
||||
});
|
||||
}
|
||||
function updateSongDatabase(wait_dialog) {
|
||||
$.ajax({
|
||||
type: 'GET',
|
||||
@ -179,13 +65,5 @@ table td:nth-child(2) {
|
||||
}
|
||||
});
|
||||
}
|
||||
function TableActions (value, row, index) {
|
||||
return "<button type=\"button\" class=\"btn btn-success\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Als gesungen markieren\" onclick=\"markEntryAsSung("+row.ID+")\"><i class=\"fas fa-check\"></i></button> <button type=\"button\" class=\"btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Eintrag löschen\" onclick=\"confirmDeleteEntry('"+row.Name+"',"+row.ID+")\"><i class=\"fas fa-trash\"></i></button>";
|
||||
}
|
||||
function getIdSelections() {
|
||||
return $.map($("#entrytable").bootstrapTable('getSelections'), function (row) {
|
||||
return row.ID
|
||||
})
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
Loading…
x
Reference in New Issue
Block a user