Begin migrating backend to MongoDB

backend/app/database.py

@@ -2,12 +2,14 @@
 
 import sqlite3
 import pandas
+import pymongo
+from bson.regex import Regex
 from io import StringIO
 
-song_table  = "songs"
+db_name = "karaoqueue"
-entry_table = "entries"
+song_collection_name  = "songs"
-index_label = "Id"
+entry_collection_name = "entries"
-done_table  = "done_songs"
+playback_collection_name  = "played"
 
 
 def dict_factory(cursor, row):
@@ -16,102 +18,99 @@ def dict_factory(cursor, row):
         d[col[0]] = row[idx]
     return d
 
-def open_db():
+def open_db_client():
-    conn = sqlite3.connect("data/test.db")
+    mongoClient = pymongo.MongoClient("mongodb://localhost:27017")
-    conn.execute('PRAGMA encoding = "UTF-8";')
+    return mongoClient
-    return conn
 
 def import_songs(song_csv):
     print("Start importing Songs...")
-    df = pandas.read_csv(StringIO(song_csv), sep=';')
+    client = open_db_client()
-    conn = open_db()
+    db = client[db_name]    
-    cur = conn.cursor()
+    if not song_collection_name in db.list_collection_names():
-    df.to_sql(song_table, conn, if_exists='replace',
+        songsCollection = db[song_collection_name]
-              index=False)
+        songsCollection.create_index("karafun_id", unique=True)
-    cur.execute("SELECT Count(Id) FROM songs")
+        songsCollection.create_index([("title","text"),("artist","text")])
-    num_songs = cur.fetchone()[0]
+    else:
-    conn.close()
+        songsCollection = db[song_collection_name]
+    
+    def f(x): return (x.split(","))
+
+    df = pandas.read_csv(StringIO(song_csv), sep=';',
+                         engine='python', parse_dates=["Date Added"])
+    df.Styles = df.Styles.apply(f, convert_dtype=True)
+    df.Languages = df.Languages.apply(f, convert_dtype=True)
+    df.Duo = df.Duo.astype('bool')
+    df.Explicit = df.Explicit.astype('bool')
+    df.columns = map(str.lower, df.columns)
+    df.rename(columns={'id': 'karafun_id'}, inplace=True)
+    num_songs = df.shape[0]
+    song_dict = df.to_dict('records')
+    try:
+        songsCollection.insert_many(song_dict)
+    except pymongo.errors.BulkWriteError as bwe:
+        return(bwe.details)
+    finally:
+        client.close()
     print("Imported songs ({} in Database)".format(num_songs))
-    return("Imported songs ({} in Database)".format(num_songs)) 
+    return("Imported songs ({} in Database)".format(num_songs))     
-
-def create_entry_table():
-    conn = open_db()
-    conn.execute('CREATE TABLE IF NOT EXISTS '+entry_table +
-                 ' (ID INTEGER PRIMARY KEY NOT NULL, Song_Id INTEGER NOT NULL, Name VARCHAR(255))')
-    conn.close()
-
-
-def create_done_song_table():
-    conn = open_db()
-    conn.execute('CREATE TABLE IF NOT EXISTS '+done_table +
-                 ' (Song_Id INTEGER PRIMARY KEY NOT NULL,  Plays INTEGER)')
-    conn.close()
-
-def create_song_table():
-    conn = open_db()
-    conn.execute("CREATE TABLE IF NOT EXISTS \""+song_table+"""\" (
-        "Id" INTEGER,
-        "Title" TEXT,
-        "Artist" TEXT,
-        "Year" INTEGER,
-        "Duo" INTEGER,
-        "Explicit" INTEGER,
-        "Date Added" TEXT,
-        "Styles" TEXT,
-        "Languages" TEXT
-    )""")
-    conn.close()
-
-def create_list_view():
-    conn = open_db()
-    conn.execute("""CREATE VIEW IF NOT EXISTS [Liste] AS
-                 SELECT Name, Title, Artist, entries.Id, songs.Id
-                 FROM entries, songs
-                 WHERE entries.Song_Id=songs.Id""")
-    conn.close()
-
-
-def create_done_song_view():
-    conn = open_db()
-    conn.execute("""CREATE VIEW IF NOT EXISTS [Abspielliste] AS
-                 SELECT Artist || \" - \" || Title AS Song, Plays AS Wiedergaben
-                 FROM songs, done_songs
-                 WHERE done_songs.Song_Id=songs.Id""")
-    conn.close()
 
 def get_list():
-    conn = open_db()
+    client = open_db_client()
-    conn.row_factory = sqlite3.Row
+    db = client[db_name]
-    cur = conn.cursor()
+    collection = db[entry_collection_name]
-    cur.execute("SELECT * FROM Liste")
-    return cur.fetchall()
 
+    query = {}
+    cursor = collection.find()
+
+    result = {}
+
+    try:
+        for doc in cursor:
+            result += doc
+    finally:
+        client.close()
 
 def get_played_list():
-    conn = open_db()
+    conn = open_db_client()
     cur = conn.cursor()
     cur.execute("SELECT * FROM Abspielliste")
     return cur.fetchall()
 
 def get_song_list():
-    conn =open_db()
+    conn =open_db_client()
     cur = conn.cursor()
     cur.execute("SELECT Artist || \" - \" || Title AS Song, Id FROM songs;")
     return cur.fetchall()
 
 def get_song_completions(input_string):
-    conn = open_db()
+    client = open_db_client()
-    conn.row_factory = dict_factory
+    db = client[db_name]
-    cur = conn.cursor()
+    collection = db[song_collection_name]
-    # Don't look, it burns...
+
-    prepared_string = "%{0}%".format(input_string).upper()  # "Test" -> "%TEST%"
+    cursor = collection.find({'$text': {'$search': input_string}}, {'_txtscr': {'$meta': 'textScore'}}, limit=30).sort([('_txtscr', {'$meta': 'textScore'})])
-    print(prepared_string)
+
-    cur.execute(
+    result = []
-        "SELECT * FROM songs WHERE REPLACE(REPLACE(REPLACE(REPLACE(UPPER( Title ),'ö','Ö'),'ü','Ü'),'ä','Ä'),'ß','ẞ') LIKE (?) LIMIT 20;", (prepared_string,))
+
-    return cur.fetchall()
+    try:
+        for doc in cursor:
+            tmpdoc = doc
+            tmpdoc["_id"] = str(tmpdoc["_id"])
+            result.append(doc)
+    finally:
+        client.close()
+    # conn = open_db_client()
+    # conn.row_factory = dict_factory
+    # cur = conn.cursor()
+    # # Don't look, it burns...
+    # prepared_string = "%{0}%".format(input_string).upper()  # "Test" -> "%TEST%"
+    # print(prepared_string)
+    # cur.execute(
+    #     "SELECT * FROM songs WHERE REPLACE(REPLACE(REPLACE(REPLACE(UPPER( Title ),'ö','Ö'),'ü','Ü'),'ä','Ä'),'ß','ẞ') LIKE (?) LIMIT 20;", (prepared_string,))
+    print(result)
+    return result
 
 def add_entry(name,song_id):
-    conn = open_db()
+    conn = open_db_client()
     cur = conn.cursor()
     cur.execute(
         "INSERT INTO entries (Song_Id,Name) VALUES(?,?);", (song_id,name))
@@ -120,7 +119,7 @@ def add_entry(name,song_id):
     return
 
 def add_sung_song(entry_id):
-    conn = open_db()
+    conn = open_db_client()
     cur = conn.cursor()
     cur.execute("""SELECT Song_Id FROM entries WHERE Id=?""",(entry_id,))
     song_id = cur.fetchone()[0]
@@ -136,7 +135,7 @@ def add_sung_song(entry_id):
     return True
 
 def clear_played_songs():
-    conn = open_db()
+    conn = open_db_client()
     cur = conn.cursor()
     cur.execute("DELETE FROM done_songs")
     conn.commit()
@@ -144,7 +143,7 @@ def clear_played_songs():
     return True
 
 def delete_entry(id):
-    conn = open_db()
+    conn = open_db_client()
     cur = conn.cursor()
     cur.execute("DELETE FROM entries WHERE id=?",(id,))
     conn.commit()
@@ -157,7 +156,7 @@ def delete_entries(ids):
     for x in ids:
         idlist.append( (x,) )
     try:
-        conn = open_db()
+        conn = open_db_client()
         cur = conn.cursor()
         cur.executemany("DELETE FROM entries WHERE id=?", idlist)
         conn.commit()
@@ -167,7 +166,7 @@ def delete_entries(ids):
         return -1
 
 def delete_all_entries():
-    conn = open_db()
+    conn = open_db_client()
     cur = conn.cursor()
     cur.execute("DELETE FROM entries")
     conn.commit()

backend/app/helpers.py

@@ -2,6 +2,8 @@ import requests
 from bs4 import BeautifulSoup
 import json
 import os
+import datetime
+import bson.objectid
 
 data_directory = "data"
 config_file = data_directory+"/config.json"
@@ -36,4 +38,11 @@ def setup_config(app):
             json.dump(config, handle, indent=4, sort_keys=True)
         print("Wrote new config")
     app.config['BASIC_AUTH_USERNAME'] = config['username']
-    app.config['BASIC_AUTH_PASSWORD'] = config['password']
+    app.config['BASIC_AUTH_PASSWORD'] = config['password']
+
+
+def serialization_helper(obj):
+    if isinstance(obj, datetime.datetime):
+        return obj.isoformat()
+    elif isinstance(obj, bson.objectid.ObjectId):
+        return str(obj)

backend/app/main.py

@@ -5,7 +5,9 @@ import database
 import data_adapters
 import os, errno
 import json
+
 from flask_basicauth import BasicAuth
+from pprint import pprint
 app = Flask(__name__, static_url_path='/static')
 
 CORS(app)
@@ -55,7 +57,7 @@ def songs():
 @app.route("/api/songs/update")
 @basic_auth.required
 def update_songs():
-    database.delete_all_entries()
+#    database.delete_all_entries()
     status = database.import_songs(helpers.get_songs(helpers.get_catalog_url()))
     print(status)
     return Response('{"status": "%s" }' % status, mimetype='text/json')
@@ -67,7 +69,7 @@ def get_song_completions(input_string=""):
     if input_string!="":
         print(input_string)
         list = database.get_song_completions(input_string=input_string)
-        return Response(json.dumps(list).encode('utf-8'), mimetype='application/json')
+        return Response(json.dumps(list, default=helpers.serialization_helper).encode('utf-8'), mimetype='application/json')
 #        return Response(json.dumps(list, ensure_ascii=False).encode('utf-8'), mimetype='text/json')
 
     else:
@@ -146,11 +148,6 @@ def admin():
 @app.before_first_request
 def activate_job():
     helpers.create_data_directory()
-    database.create_entry_table()
-    database.create_song_table()
-    database.create_done_song_table()
-    database.create_list_view()
-    database.create_done_song_view()
     helpers.setup_config(app)
 
 

backend/app/templates/main_admin.html

@@ -18,73 +18,11 @@ table td:nth-child(2) {
             aktualisieren</button>
         <input id="entryToggle" type="checkbox" class="topbutton" data-toggle="toggle" data-on="Eintragen erlaubt" data-off="Eintragen deaktiviert" data-onstyle="success" data-offstyle="danger">
     </div>
-    <table class="table entries"
-            id="entrytable"
-            data-toggle="table"
-            data-search="true"
-            data-show-columns="true"
-            data-show-toggle="true"
-            data-multiple-select-row="true"
-            data-click-to-select="true"
-            data-toolbar="#toolbar"
-            data-pagination="true"
-            data-show-extended-pagination="true"
-            data-classes="table table-hover"
-            data-url="/api/queue"
-            data-show-refresh="true"
-            data-auto-refresh="true"
-            data-auto-refresh-interval="10">
-        <thead>
-            <tr>
-                <th data-field="state" data-checkbox="true"></th>
-                <th scope="col" data-field="Name">Name</th>
-                <th scope="col" data-field="Title">Song</th>
-                <th scope="col" data-field="Artist">Künstler</th>
-                <th scope="col" data-formatter="TableActions">Aktionen</th>
-            </tr>
-        </thead>
-    </table>
     <a name="end"></a>
 </div>
 {% endblock %}
 {% block extrajs %}
 <script>
-    $(function () {
-        $('#entryToggle').change(function() {
-            $.ajax({url: "/api/entries/accept/"+($('#entryToggle').is(":checked") ? "1" : "0"), complete: setTimeout(refreshEntryToggle, 1000)});
-        })
-        refreshEntryToggle()
-        $("#entrytable").bootstrapTable().on('load-success.bs.table', function() {
-            $('[data-toggle="tooltip"]').tooltip()
-        })
-    })
-    function confirmDeleteEntry(name, entry_id) {
-        bootbox.confirm("Wirklich den Eintrag von "+name+" löschen?", function(result){
-            if (result) {
-                deleteEntry(entry_id)
-            }
-        })
-    }
-    function confirmDeleteSelectedEntries() {
-        bootbox.confirm({
-            message: "Wirklich gewählte Eintragungen löschen?", 
-            buttons: {
-                confirm: {
-                    label: 'Ja',
-                    className: 'btn btn-danger'
-                },
-                cancel: {
-                    label: 'Nein',
-                    className: 'btn btn-secondary'
-                }
-            },
-            callback: function(result){
-                if (result) {
-                    DeleteSelectedEntries(getIdSelections())
-                }
-            }
-        })
-    }
     function confirmUpdateSongDatabase() {
         bootbox.confirm({
             message: "Wirklich die Song-Datenbank aktualisieren?<br>Dies lädt die Aktuelle Song-Liste von <a href='https://www.karafun.de/karaoke-song-list.html'>KaraFun</a> herunter, <b>und wird alle Eintragungen löschen!</b>",
@@ -110,58 +48,6 @@ table td:nth-child(2) {
             }
         })
     }
-    function refreshEntryToggle() {
-        $.getJSON("/api/entries/accept", (data) => {
-            if (data["value"]!=$('#entryToggle').is(":checked")) {
-                if(data["value"]==1) {
-                $('#entryToggle').bootstrapToggle('on')
-                }
-                else {
-                    $('#entryToggle').bootstrapToggle('off')
-                }
-            }
-        })
-    }
-    function deleteEntry(entry_id) {
-       $.ajax({
-        type: 'GET',
-        url: '/api/entries/delete/'+entry_id,
-        contentType: "application/json",
-        dataType: 'json',
-        async: false
-       });
-       $("#entrytable").bootstrapTable('refresh')
-    
-    }
-    function markEntryAsSung(entry_id) {
-       $.ajax({
-        type: 'GET',
-        url: '/api/entries/mark_sung/'+entry_id,
-        contentType: "application/json",
-        dataType: 'json',
-        async: false
-       });
-       $("#entrytable").bootstrapTable('refresh')
-    
-    }
-    function DeleteSelectedEntries(ids) {
-        $.ajax({
-            type: 'POST',
-            url: '/api/entries/delete',
-            data: JSON.stringify(ids), // or JSON.stringify ({name: 'jonas'}),
-            error: function() {
-                bootbox.alert({
-                    message: "Fehler beim Löschen der Eintragungen.",
-                })
-            },
-            success: function() {
-                $("#entrytable").bootstrapTable('refresh')
-            
-            },
-            contentType: "application/json",
-            dataType: 'json'
-        });
-    }
     function updateSongDatabase(wait_dialog) {
         $.ajax({
             type: 'GET',
@@ -179,13 +65,5 @@ table td:nth-child(2) {
             }
         });
     }
-    function TableActions (value, row, index) {
-        return "<button type=\"button\" class=\"btn btn-success\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Als gesungen markieren\" onclick=\"markEntryAsSung("+row.ID+")\"><i class=\"fas fa-check\"></i></button>&nbsp;<button type=\"button\" class=\"btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Eintrag löschen\" onclick=\"confirmDeleteEntry('"+row.Name+"',"+row.ID+")\"><i class=\"fas fa-trash\"></i></button>";
-    }
-    function getIdSelections() {
-        return $.map($("#entrytable").bootstrapTable('getSelections'), function (row) {
-            return row.ID
-        })
-    }
 </script>
 {% endblock %}