Merge pull request #39 from PhoenixTwoFive/feature/legacy/mariadb_database

Feature/legacy/mariadb database

.vscode/launch.json

@@ -5,7 +5,7 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "preLaunchTask": "versiondump",
+            "preLaunchTask": "mariadb",
             "name": "Python: Flask",
             "type": "python",
             "cwd": "${workspaceFolder}/backend",
@@ -14,7 +14,8 @@
             "env": {
                 "FLASK_APP": "backend/app.py",
                 "FLASK_ENV": "development",
-                "FLASK_DEBUG": "1"
+                "FLASK_DEBUG": "1", 
+                "DBSTRING": "mysql://devuser:devpw@127.0.0.1:3306/karaoqueue"
             },
             "args": [
                 "run",
@@ -24,7 +25,7 @@
             "jinja": true
         },
         {
-            "preLaunchTask": "versiondump",
+            "preLaunchTask": "mariadb",
             "name": "Python: Flask (with reload)",
             "type": "python",
             "cwd": "${workspaceFolder}/backend",
@@ -42,7 +43,7 @@
             "jinja": true
         },
         {
-            "preLaunchTask": "versiondump",
+            "preLaunchTask": "mariadb",
             "name": "Python: Flask (with reload, externally reachable)",
             "type": "python",
             "cwd": "${workspaceFolder}/backend",
@@ -61,7 +62,7 @@
             "jinja": true
         },
         {
-            "preLaunchTask": "versiondump",
+            "preLaunchTask": "mariadb",
             "name": "Python: Flask (externally reachable)",
             "type": "python",
             "cwd": "${workspaceFolder}/backend",

.vscode/tasks.json

@@ -8,6 +8,13 @@
             "type": "shell",
             "command": "echo \"$(git rev-parse --abbrev-ref HEAD)-$(git describe)\"> ${workspaceFolder}/backend/.version",
             "problemMatcher": []
+        },
+        {
+            "label": "mariadb",
+            "type": "shell",
+            "command": "docker run --rm --name some-mariadb --env MARIADB_USER=devuser --env MARIADB_PASSWORD=devpw --env MARIADB_ROOT_PASSWORD=devrootpw --env MARIADB_DATABASE=karaoqueue -p 3306:3306 mariadb:latest",
+            "isBackground": true,
+            "activeOnStart": false
         }
     ]
 }

Dockerfile

@@ -8,4 +8,4 @@ RUN pip install Flask-BasicAuth
 
 RUN pip install bs4
 
-COPY ./app /app
+COPY ./backend /app

backend/app.py

@@ -1,4 +1,5 @@
-from flask import Flask, render_template, Response, abort, request, redirect, send_from_directory
+from flask import Flask, render_template, abort, request, redirect, send_from_directory, jsonify
+from flask.wrappers import Request, Response
 import helpers
 import database
 import data_adapters
@@ -9,7 +10,7 @@ from helpers import nocache
 app = Flask(__name__, static_url_path='/static')
 
 basic_auth = BasicAuth(app)
-accept_entries = False
+accept_entries = True
 
 @app.route("/")
 def home():
@@ -81,12 +82,12 @@ def settings():
 def settings_post():
     entryquota = request.form.get("entryquota")
     maxqueue = request.form.get("maxqueue")
-    if entryquota.isnumeric() and int(entryquota) > 0:
+    if entryquota.isnumeric() and int(entryquota) > 0:  # type: ignore
-        app.config['ENTRY_QUOTA'] = int(entryquota)
+        app.config['ENTRY_QUOTA'] = int(entryquota)  # type: ignore
     else:
         abort(400)
-    if maxqueue.isnumeric and int(maxqueue) > 0:
+    if maxqueue.isnumeric and int(maxqueue) > 0:  # type: ignore
-        app.config['MAX_QUEUE'] = int(maxqueue)
+        app.config['MAX_QUEUE'] = int(maxqueue)  # type: ignore
     else:
         abort(400)
 
@@ -131,8 +132,8 @@ def get_song_completions(input_string=""):
     input_string = request.args.get('search', input_string)
     if input_string != "":
         print(input_string)
-        list = database.get_song_completions(input_string=input_string)
+        result = [list(x) for x in database.get_song_completions(input_string=input_string)]
-        return Response(json.dumps(list, ensure_ascii=False).encode('utf-8'), mimetype='text/json')
+        return jsonify(result)
 
     else:
         return 400
@@ -229,6 +230,7 @@ def admin():
 
 @app.before_first_request
 def activate_job():
+    helpers.load_dbconfig(app)
     helpers.load_version(app)
     helpers.create_data_directory()
     database.create_entry_table()

backend/database.py

@@ -1,31 +1,39 @@
 # -*- coding: utf_8 -*-
 
-import sqlite3
+from email.mime import base
+from MySQLdb import Connection
+from sqlalchemy import create_engine, engine
 import pandas
 from io import StringIO
+from flask import current_app
 
 song_table = "songs"
 entry_table = "entries"
 index_label = "Id"
 done_table = "done_songs"
 
+connection = None
 
-def open_db():
+
-    conn = sqlite3.connect("/tmp/karaoqueue.db")
+def open_db() -> engine.base.Connection:
-    conn.execute('PRAGMA encoding = "UTF-8";')
+    global connection
-    return conn
+    if (not connection):
+        print(current_app.config.get("DBCONNSTRING"))
+        engine = create_engine(current_app.config.get("DBCONNSTRING"))  # type: ignore
+        connection = engine.connect()
+    # cur.execute('PRAGMA encoding = "UTF-8";')
+    return connection
 
 
 def import_songs(song_csv):
     print("Start importing Songs...")
     df = pandas.read_csv(StringIO(song_csv), sep=';')
     conn = open_db()
-    cur = conn.cursor()
     df.to_sql(song_table, conn, if_exists='replace',
               index=False)
-    cur.execute("SELECT Count(Id) FROM songs")
+    cur = conn.execute("SELECT Count(Id) FROM songs")
-    num_songs = cur.fetchone()[0]
+    num_songs = cur.fetchone()[0] # type: ignore
-    conn.close()
+    # conn.close()
     print("Imported songs ({} in Database)".format(num_songs))
     return("Imported songs ({} in Database)".format(num_songs))
 
@@ -33,158 +41,146 @@ def import_songs(song_csv):
 def create_entry_table():
     conn = open_db()
     conn.execute('CREATE TABLE IF NOT EXISTS '+entry_table +
-                 ' (ID INTEGER PRIMARY KEY NOT NULL, Song_Id INTEGER NOT NULL, Name VARCHAR(255), Client_Id VARCHAR(36), Transferred INTEGER DEFAULT 0)')
+                 ' (ID INTEGER PRIMARY KEY NOT NULL AUTO_INCREMENT, Song_Id INTEGER NOT NULL, Name VARCHAR(255), Client_Id VARCHAR(36), Transferred INTEGER DEFAULT 0)')
-    conn.close()
+    # conn.close()
 
 
 def create_done_song_table():
     conn = open_db()
     conn.execute('CREATE TABLE IF NOT EXISTS '+done_table +
                  ' (Song_Id INTEGER PRIMARY KEY NOT NULL,  Plays INTEGER)')
-    conn.close()
+    # conn.close()
 
 
 def create_song_table():
     conn = open_db()
-    conn.execute("CREATE TABLE IF NOT EXISTS \""+song_table+"""\" (
+    conn.execute("CREATE TABLE IF NOT EXISTS `"+song_table+"""` (
-        "Id" INTEGER,
+        `Id` INTEGER,
-        "Title" TEXT,
+        `Title` TEXT,
-        "Artist" TEXT,
+        `Artist` TEXT,
-        "Year" INTEGER,
+        `Year` INTEGER,
-        "Duo" INTEGER,
+        `Duo` INTEGER,
-        "Explicit" INTEGER,
+        `Explicit` INTEGER,
-        "Date Added" TEXT,
+        `Date Added` TEXT,
-        "Styles" TEXT,
+        `Styles` TEXT,
-        "Languages" TEXT
+        `Languages` TEXT
     )""")
-    conn.close()
+    # conn.close()
 
 
 def create_list_view():
     conn = open_db()
-    conn.execute("""CREATE VIEW IF NOT EXISTS [Liste] AS
+    conn.execute("""CREATE OR REPLACE VIEW `Liste` AS
-                 SELECT Name, Title, Artist, entries.Id, songs.Id, entries.Transferred
+                 SELECT Name, Title, Artist, entries.Id AS entry_ID, songs.Id AS song_ID, entries.Transferred
                  FROM entries, songs
                  WHERE entries.Song_Id=songs.Id""")
-    conn.close()
+    # conn.close()
 
 
 def create_done_song_view():
     conn = open_db()
-    conn.execute("""CREATE VIEW IF NOT EXISTS [Abspielliste] AS
+    conn.execute("""CREATE OR REPLACE VIEW `Abspielliste` AS
-                 SELECT Artist || \" - \" || Title AS Song, Plays AS Wiedergaben
+                 SELECT CONCAT(Artist," - ", Title) AS Song, Plays AS Wiedergaben
                  FROM songs, done_songs
                  WHERE done_songs.Song_Id=songs.Id""")
-    conn.close()
+    # conn.close()
 
 
 def get_list():
     conn = open_db()
-    conn.row_factory = sqlite3.Row
+    cur = conn.execute("SELECT * FROM Liste")
-    cur = conn.cursor()
-    cur.execute("SELECT * FROM Liste")
     return cur.fetchall()
 
 
 def get_played_list():
     conn = open_db()
-    cur = conn.cursor()
+    cur = conn.execute("SELECT * FROM Abspielliste")
-    cur.execute("SELECT * FROM Abspielliste")
     return cur.fetchall()
 
 
 def get_song_list():
     conn = open_db()
-    cur = conn.cursor()
+    cur = conn.execute(
-    cur.execute("SELECT Artist || \" - \" || Title AS Song, Id FROM songs;")
+        "SELECT Artist || \" - \" || Title AS Song, Id FROM songs;")
     return cur.fetchall()
 
 
 def get_song_completions(input_string):
     conn = open_db()
-    cur = conn.cursor()
     # Don't look, it burns...
     prepared_string = "%{0}%".format(
         input_string).upper()  # "Test" -> "%TEST%"
     print(prepared_string)
-    cur.execute(
+    cur = conn.execute(
-        "SELECT Title || \" - \" || Artist AS Song, Id FROM songs WHERE REPLACE(REPLACE(REPLACE(REPLACE(UPPER( SONG ),'ö','Ö'),'ü','Ü'),'ä','Ä'),'ß','ẞ') LIKE (?) LIMIT 20;", (prepared_string,))
+        "SELECT CONCAT(Artist,\" - \",Title) AS Song, Id FROM songs WHERE CONCAT(Artist,\" - \",Title) LIKE (%s) LIMIT 20;", [prepared_string])
     return cur.fetchall()
 
 
 def add_entry(name, song_id, client_id):
     conn = open_db()
-    cur = conn.cursor()
+    conn.execute(
-    cur.execute(
+        "INSERT INTO entries (Song_Id,Name,Client_Id) VALUES(%s,%s,%s);", (song_id, name, client_id))
-        "INSERT INTO entries (Song_Id,Name,Client_Id) VALUES(?,?,?);", (song_id, name, client_id))
+    # conn.close()
-    conn.commit()
-    conn.close()
     return
 
 
 def add_sung_song(entry_id):
     conn = open_db()
-    cur = conn.cursor()
+    cur = conn.execute(
-    cur.execute("""SELECT Song_Id FROM entries WHERE Id=?""", (entry_id,))
+        """SELECT Song_Id FROM entries WHERE Id=%s""", (entry_id,))
-    song_id = cur.fetchone()[0]
+    song_id = cur.fetchone()[0] # type: ignore
-    cur.execute("""INSERT OR REPLACE INTO done_songs (Song_Id, Plays)
+    conn.execute("""INSERT INTO done_songs (Song_Id, Plays) VALUES("""+str(song_id)+""",1) ON DUPLICATE KEY UPDATE Plays=Plays + 1;""")
-                VALUES("""+str(song_id)+""",
+# SQLite bullshittery
-                       COALESCE(
+#    conn.execute("""REPLACE INTO done_songs (Song_Id, Plays)
-                           (SELECT Plays FROM done_songs
+#                VALUES("""+str(song_id)+""",
-                            WHERE Song_Id="""+str(song_id)+"), 0) + 1)"
+#                       COALESCE(
-                )
+#                           (SELECT Plays FROM done_songs
-    conn.commit()
+#                            WHERE Song_Id="""+str(song_id)+"), 0) + 1)"
+#                 )
     delete_entry(entry_id)
-    conn.close()
+    # conn.close()
     return True
 
 
 def toggle_transferred(entry_id):
     conn = open_db()
-    cur = conn.cursor()
+    cur = conn.execute(
-    cur.execute("SELECT Transferred FROM entries WHERE ID =?", (entry_id,))
+        "SELECT Transferred FROM entries WHERE ID =%s", (entry_id,))
     marked = cur.fetchall()[0][0]
     if(marked == 0):
-        cur.execute(
+        conn.execute(
-            "UPDATE entries SET Transferred = 1 WHERE ID =?", (entry_id,))
+            "UPDATE entries SET Transferred = 1 WHERE ID =%s", (entry_id,))
     else:
-        cur.execute(
+        conn.execute(
-            "UPDATE entries SET Transferred = 0 WHERE ID =?", (entry_id,))
+            "UPDATE entries SET Transferred = 0 WHERE ID =%s", (entry_id,))
-    conn.commit()
+    # conn.close()
-    conn.close()
     return True
 
 
 def check_entry_quota(client_id):
     conn = open_db()
-    cur = conn.cursor()
+    cur = conn.execute(
-    cur.execute(
+        "SELECT Count(*) FROM entries WHERE entries.Client_Id = %s", (client_id,))
-        "SELECT Count(*) FROM entries WHERE entries.Client_Id = ?", (client_id,))
     return cur.fetchall()[0][0]
 
 
 def check_queue_length():
     conn = open_db()
-    cur = conn.cursor()
+    cur = conn.execute("SELECT Count(*) FROM entries")
-    cur.execute("SELECT Count(*) FROM entries")
     return cur.fetchall()[0][0]
 
 
 def clear_played_songs():
     conn = open_db()
-    cur = conn.cursor()
+    conn.execute("DELETE FROM done_songs")
-    cur.execute("DELETE FROM done_songs")
+    # conn.close()
-    conn.commit()
-    conn.close()
     return True
 
 
 def delete_entry(id):
     conn = open_db()
-    cur = conn.cursor()
+    conn.execute("DELETE FROM entries WHERE id=%s", (id,))
-    cur.execute("DELETE FROM entries WHERE id=?", (id,))
+    # conn.close()
-    conn.commit()
-    conn.close()
     return True
 
 
@@ -194,19 +190,15 @@ def delete_entries(ids):
         idlist.append((x,))
     try:
         conn = open_db()
-        cur = conn.cursor()
+        cur = conn.execute("DELETE FROM entries WHERE id=%s", idlist)
-        cur.executemany("DELETE FROM entries WHERE id=?", idlist)
+        # conn.close()
-        conn.commit()
-        conn.close()
         return cur.rowcount
-    except sqlite3.Error as error:
+    except Exception as error:
         return -1
 
 
 def delete_all_entries():
     conn = open_db()
-    cur = conn.cursor()
+    conn.execute("DELETE FROM entries")
-    cur.execute("DELETE FROM entries")
+    # conn.close()
-    conn.commit()
-    conn.close()
     return True

backend/helpers.py

@@ -37,7 +37,7 @@ def check_config_exists():
 
 def load_version(app):
     if os.environ.get("SOURCE_VERSION"):
-        app.config['VERSION'] = os.environ.get("SOURCE_VERSION")[0:7]
+        app.config['VERSION'] = os.environ.get("SOURCE_VERSION")[0:7] # type: ignore
     elif os.path.isfile(".version"):
         with open('.version', 'r') as file:
             data = file.read().replace('\n', '')
@@ -48,7 +48,37 @@ def load_version(app):
     else:
         app.config['VERSION'] = ""
         
+def load_dbconfig(app):
+    if os.environ.get("FLASK_ENV") == "development":
+        app.config['DBCONNSTRING'] = os.environ.get("DBSTRING")
+    else:
+        if os.environ.get("DEPLOYMENT_PLATFORM") == "Heroku":    
+            if os.environ.get("JAWSDB_MARIA_URL"):
+                app.config['DBCONNSTRING'] = os.environ.get("JAWSDB_MARIA_URL")
+            else:
+                app.config['DBCONNSTRING'] = ""
+        if os.environ.get("DEPLOYMENT_PLATFORM") == "Docker": 
+            if os.environ.get("DBSTRING"):
+                app.config['DBCONNSTRING'] = os.environ.get("DBSTRING")
+            else:
+                app.config['DBCONNSTRING'] = ""   
+        elif os.path.isfile(".dbconn"):
+            with open('.dbconn', 'r') as file:
+                data = file.read().replace('\n', '')
+                if data:
+                    app.config['DBCONNSTRING'] = data
+                else:
+                    app.config['DBCONNSTRING'] = ""
+        else:
+            app.config['DBCONNSTRING'] = ""
+
 def setup_config(app):
+    if os.environ.get("DEPLOYMENT_PLATFORM") == "Docker":
+        app.config['BASIC_AUTH_USERNAME'] = os.environ.get('BASIC_AUTH_USERNAME')
+        app.config['BASIC_AUTH_PASSWORD'] = os.environ.get('BASIC_AUTH_PASSWORD')
+        app.config['ENTRY_QUOTA'] = os.environ.get('ENTRY_QUOTA')
+        app.config['MAX_QUEUE'] = os.environ.get('MAX_QUEUE')
+    else:
         if check_config_exists():
             config = json.load(open(config_file))
             with open(config_file, 'r') as handle:
@@ -70,7 +100,7 @@ def nocache(view):
     @wraps(view)
     def no_cache(*args, **kwargs):
         response = make_response(view(*args, **kwargs))
-        response.headers['Last-Modified'] = datetime.now()
+        response.headers['Last-Modified'] = datetime.now()  # type: ignore
         response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
         response.headers['Pragma'] = 'no-cache'
         response.headers['Expires'] = '-1'

backend/requirements.txt

@@ -3,3 +3,5 @@ pandas
 Flask-BasicAuth
 bs4
 gunicorn
+SQLAlchemy
+mysqlclient

backend/templates/base.html

@@ -75,7 +75,7 @@
             {% endif %}
             <!--<a href="https://github.com/PhoenixTwoFive/karaoqueue"
                 class="ml-1 mr-1"><i class="fab fa-github mr-1"></i><span>Github</span></a>-->
-            <span class="text-muted">KaraoQueue {{karaoqueue_version}} -&nbsp;<span>&copy</span>&nbsp;2019-21 - Phillip
+            <span class="text-muted"><a class="text-muted" href="https://github.com/PhoenixTwoFive/karaoqueue">KaraoQueue</a> {{karaoqueue_version}} -&nbsp;2019-22 - Phillip
                 Kühne</span>
         </div>
     </footer>

backend/templates/main_admin.html

@@ -52,10 +52,10 @@ table td:nth-child(2) {
 {% block extrajs %}
 <script>
     $(function () {
+        refreshEntryToggle()
         $('#entryToggle').change(function() {
             $.ajax({url: "/api/entries/accept/"+($('#entryToggle').is(":checked") ? "1" : "0"), complete: setTimeout(refreshEntryToggle, 1000)});
         })
-        refreshEntryToggle()
         $("#entrytable").bootstrapTable().on('load-success.bs.table', function() {
             $('[data-toggle="tooltip"]').tooltip()
         })
@@ -195,17 +195,19 @@ table td:nth-child(2) {
         });
     }
     function TableActions (value, row, index) {
+        console.log("Value: "+value+", Row: "+row+", Index: "+index)
+        console.log(row)
         let outerHTML = ""
         if (row.Transferred==1) {
-            outerHTML = "<button type=\"button\" class=\"btn btn-default\" onclick=\"markEntryAsTransferred("+row.ID+")\"><i class=\"fas fa-backward\"></i></button>&nbsp;<button type=\"button\" class=\"btn btn-success\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Als gesungen markieren\" onclick=\"markEntryAsSung("+row.ID+")\"><i class=\"fas fa-check\"></i></button>&nbsp;<button type=\"button\" class=\"btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Eintrag löschen\" onclick=\"confirmDeleteEntry('"+row.Name+"',"+row.ID+")\"><i class=\"fas fa-trash\"></i></button>";
+            outerHTML = "<button type=\"button\" class=\"btn btn-default\" onclick=\"markEntryAsTransferred("+row.entry_ID+")\"><i class=\"fas fa-backward\"></i></button>&nbsp;<button type=\"button\" class=\"btn btn-success\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Als gesungen markieren\" onclick=\"markEntryAsSung("+row.entry_ID+")\"><i class=\"fas fa-check\"></i></button>&nbsp;<button type=\"button\" class=\"btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Eintrag löschen\" onclick=\"confirmDeleteEntry('"+row.Name+"',"+row.entry_ID+")\"><i class=\"fas fa-trash\"></i></button>";
         } else {
-            outerHTML = "<button type=\"button\" class=\"btn btn-info\" onclick=\"markEntryAsTransferred("+row.ID+")\"><i class=\"fas fa-exchange-alt\"></i></button>&nbsp;<button type=\"button\" class=\"btn btn-success\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Als gesungen markieren\" onclick=\"markEntryAsSung("+row.ID+")\"><i class=\"fas fa-check\"></i></button>&nbsp;<button type=\"button\" class=\"btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Eintrag löschen\" onclick=\"confirmDeleteEntry('"+row.Name+"',"+row.ID+")\"><i class=\"fas fa-trash\"></i></button>";
+            outerHTML = "<button type=\"button\" class=\"btn btn-info\" onclick=\"markEntryAsTransferred("+row.entry_ID+")\"><i class=\"fas fa-exchange-alt\"></i></button>&nbsp;<button type=\"button\" class=\"btn btn-success\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Als gesungen markieren\" onclick=\"markEntryAsSung("+row.entry_ID+")\"><i class=\"fas fa-check\"></i></button>&nbsp;<button type=\"button\" class=\"btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"top\" title=\"Eintrag löschen\" onclick=\"confirmDeleteEntry('"+row.Name+"',"+row.entry_ID+")\"><i class=\"fas fa-trash\"></i></button>";
         }
         return outerHTML;
     }
     function getIdSelections() {
         return $.map($("#entrytable").bootstrapTable('getSelections'), function (row) {
-            return row.ID
+            return row.entry_ID
         })
     }
 </script>

docker-compose.prod.yml

@@ -0,0 +1,23 @@
+version: "3.9"
+services:
+  karaoqueue:
+    image: "phillipkhne/karaoqueue:latest"
+    restart: always
+    ports:
+      - "127.0.0.1:8081:80"
+    environment:
+      DEPLOYMENT_PLATFORM: Docker
+      DBSTRING: mysql://user:pass@host:3306/database
+      BASIC_AUTH_USERNAME: admin
+      BASIC_AUTH_PASSWORD: changeme
+      ENTRY_QUOTA: 3
+      MAX_QUEUE: 20
+  db:
+    image: mariadb
+    restart: always
+    environment:
+      MARIADB_ROOT_PASSWORD: dpMAZj*Mc4%FZM!V
+      MARIADB_ROOT_HOST: localhost
+      MARIADB_DATABASE: karaoqueue
+      MARIADB_USER: karaoqueue
+      MARIADB_PASSWORD: a5G@P*^tCW$$w@wE