[MOD] Verbosity levels, avatar cleanser

I have a weird bug where Parsee seems to hang on a Db request, been
tryign to figure that otu since a while but can't quite put my finger on
where.
You can have this commit, as a treat.

src/Commands/Stats.c

@@ -24,7 +24,6 @@ CommandHead(CmdStats, cmd, argp)
     BotInitialise();
 
 
-    /* TODO: Separate these into different "categories" */
     ReplySprintf("Information for %s v%s (Cytoplasm %s)",
         NAME, VERSION, CytoplasmGetVersionStr()
     );

src/Events.c

@@ -37,12 +37,10 @@ MatrixCreateMessage(char *body)
     HashMapSet(map, "msgtype", JsonValueString("m.text"));
     HashMapSet(map, "body", JsonValueString(body));
     {
-        /* TODO */
         XEP393Element *e = XEP393(body);
         text = XEP393ToXMLString(e);
         XEP393FreeElement(e);
 
-
         HashMapSet(map, "formatted_body", JsonValueString(text));
         HashMapSet(map, "format", JsonValueString("org.matrix.custom.html"));
         Free(text);

src/Glob.c

@@ -19,7 +19,6 @@ GlobMatches(char *rule, char *string)
         switch (c1)
         {
             case '*':
-                /* TODO */
                 while ((c2 = *string) && (c2 != next))
                 {
                     string++;

src/HttParsee.c

@@ -41,7 +41,6 @@ ParseeRequest(HttpServerContext *ctx, void *argp)
         JsonFree(response);
 
         response = MatrixCreateError("M_NOT_FOUND", "Route not found.");
-        /* TODO: Set a thing */
     }
 
     /* Whatever, we routed a thing. */

src/Main.c

@@ -43,7 +43,10 @@ static const Argument arguments[] =
     Arg('C', true, "file(='parsee.json')", "Sets the JSON config to use")
 
     Arg('g', false, NULL,"Generates a parsee.yaml AS file before exiting")
-    Arg('v', false, NULL,"Forces Parsee to print in a more verbose fashion")
+    Arg('v', false, NULL,
+        "Forces Parsee to print in a more verbose fashion "
+        "(-vv prints stanzas to stderr)"
+    )
     Arg('h', false, NULL,"Generates an help screen(this one!)")
 
     EndOfArgs
@@ -63,6 +66,7 @@ Main(Array *args, HashMap *env)
     char *configuration = "parsee.json";
     int xmpp = 8;
     int http = 8;
+    int verbose = 0;
 
     start = UtilTsMillis();
 
@@ -78,10 +82,7 @@ Main(Array *args, HashMap *env)
         ArgParseState state;
         char *opts = ParseeGenerateGetopt(arguments);
         int flag;
-
         ArgParseStateInit(&state);
-        /* TODO: Have a smarter way of generating the arg table 
-         * (with a list of structs, with a description and everything) */
         while ((flag = ArgParse(&state, args, opts)) != -1)
         {
             switch (flag)
@@ -105,7 +106,26 @@ Main(Array *args, HashMap *env)
                     Free(opts);
                     goto end;
                 case 'v':
-                    LogConfigLevelSet(LogConfigGlobal(), LOG_DEBUG);
+                    switch (++verbose)
+                    {
+                        case PARSEE_VERBOSE_LOG:
+                            LogConfigLevelSet(LogConfigGlobal(), LOG_DEBUG);
+                            break;
+                        case PARSEE_VERBOSE_STANZA:
+                            Log(LOG_DEBUG, "Enabling stanza printing.");
+                            break;
+                        case PARSEE_VERBOSE_COMICAL:
+                            Log(LOG_DEBUG, "What?");
+                            Log(LOG_DEBUG, "No, but like, what do you except?");
+                            Log(LOG_DEBUG, "Like do you want to log _every_ instruction?");
+                            Log(LOG_DEBUG, "Like just every single thing %s does?", NAME);
+                            Log(LOG_DEBUG, "          ( why??? )");
+                            Log(LOG_DEBUG, ".....................................");
+                            Log(LOG_DEBUG, "Argh.");
+                            Log(LOG_DEBUG, "Alright. I'll do my best.");
+                            Log(LOG_DEBUG, "Get what you paid for.");
+                            break;
+                    }
                     break;
                 case 'C':
                     if (!UtilLastModified(state.optArg))
@@ -126,6 +146,10 @@ Main(Array *args, HashMap *env)
         ParseeSetThreads(xmpp, http);
     }
 
+    if (verbose >= PARSEE_VERBOSE_COMICAL)
+    {
+        Log(LOG_DEBUG, "Loading configuration...");
+    }
     ParseeConfigLoad(configuration);
     ParseeConfigInit();
     parsee_conf = ParseeConfigGet();
@@ -142,14 +166,38 @@ Main(Array *args, HashMap *env)
     ))
     {
         Log(LOG_ERR, "Could not connect to XMPP...");
+
+        if (verbose >= PARSEE_VERBOSE_COMICAL)
+        {
+            Log(LOG_DEBUG, "Destroying component...");
+        }
         XMPPEndCompStream(jabber);
         goto end;
     }
 
     Log(LOG_NOTICE, "Creating volatile tables...");
+    if (verbose >= PARSEE_VERBOSE_COMICAL)
+    {
+        Log(LOG_DEBUG, "Initialising JID table");
+    }
     ParseeInitialiseJIDTable();
+
+    if (verbose >= PARSEE_VERBOSE_COMICAL)
+    {
+        Log(LOG_DEBUG, "Initialising OID table");
+    }
     ParseeInitialiseOIDTable();
+
+    if (verbose >= PARSEE_VERBOSE_COMICAL)
+    {
+        Log(LOG_DEBUG, "Initialising head table");
+    }
     ParseeInitialiseHeadTable();
+
+    if (verbose >= PARSEE_VERBOSE_COMICAL)
+    {
+        Log(LOG_DEBUG, "Initialising nick table");
+    }
     ParseeInitialiseNickTable();
 
     conf.port = parsee_conf->port;
@@ -158,6 +206,9 @@ Main(Array *args, HashMap *env)
     conf.handlerArgs = ParseeInitData(jabber);
     conf.handler = ParseeRequest;
 
+    Log(LOG_DEBUG, "Verbosity level: %d", verbose);
+    ((ParseeData *) conf.handlerArgs)->verbosity = verbose;
+
     Log(LOG_NOTICE, "Setting up local Matrix user...");
     if (ASRegisterUser(parsee_conf, parsee_conf->sender_localpart))
     {

src/Parsee/Config.c

@@ -109,7 +109,6 @@ PromptInteger(const char *expression, int def, ...)
     return l;
 }
 
-/* TODO: Memleaks, galore! */
 void
 ParseeConfigInit(void)
 {
@@ -120,8 +119,7 @@ ParseeConfigInit(void)
         return;
     }
 
-    /* TODO: Give the user an achievement at the end, just because they're 
+    /* TODO: Get rid of this, as parsee-config is the main way of doing it */
-     * cool. */
     Log(LOG_NOTICE, "It seems like it is the first time you have configured ");
     Log(LOG_NOTICE, "Parsee.");
     Log(LOG_NOTICE, "As such, I need to ask you a couple of questions before ");
@@ -137,7 +135,6 @@ ParseeConfigInit(void)
     config->xmpp_threads = 8;
     config->db_size = 64 MB;
 
-    /* TODO: This is NOT user friendly, and I know it! */
     config->sender_localpart = PromptString(
         "Name of the bridge bot, used for commands and bridged rooms", 
         "_parsee_bridge"
@@ -150,7 +147,7 @@ ParseeConfigInit(void)
     config->listen_as = StrDuplicate("localhost");
     config->port = PromptInteger(
         "Matrix port for the AS service to use", 
-        7642    /* proposed by Saint */
+        7642
     );
 
 
@@ -185,7 +182,6 @@ ParseeConfigInit(void)
         NULL
     );
 
-    /* TODO: Make that configurable. */
     config->server_base = StrDuplicate(config->homeserver_host);
 
     Log(LOG_NOTICE, "Done! Please look over to the parsee.yaml file, ");

src/Parsee/User.c

@@ -96,7 +96,7 @@ ParseeDecodeLocalJID(const ParseeConfig *c, char *mxid)
     data_start = jid_flags;
     while (*data_start && *data_start != '_')
     {
-        /* TODO: Make this a macro */
+        /* TODO: Get rid of this */
         if (*data_start == 'l')
         {
             plain_jid = true;
@@ -126,7 +126,7 @@ ParseeDecodeLocalMUC(const ParseeConfig *c, char *alias)
     data_start = jid_flags;
     while (*data_start && *data_start != '_')
     {
-        /* TODO: Make this a macro */
+        /* TODO: Get rid of this */
         if (*data_start == 'm')
         {
             plain_jid = true;
@@ -236,7 +236,6 @@ ParseeEncodeMXID(char *mxid)
     {
         char src = mxid[i];
 
-        /* TODO: More robust system */
         if (src <= 0x20 ||
             (src == '"' || src == '&' ||
               src == '\'' || src == '/' ||
@@ -685,7 +684,6 @@ ParseeToUnauth(ParseeData *data, char *mxc)
         return NULL;
     }
 
-    /* TODO: HTTPS */
     l = snprintf(NULL, 0, 
         PAT,
         data->config->media_base, 

src/Routes/Ping.c

@@ -1,37 +0,0 @@
-#include <Routes.h>
-
-#include <Cytoplasm/Log.h>
-
-#include <Matrix.h>
-#include <AS.h>
-
-RouteHead(RoutePing, arr, argp)
-{
-    ParseeHttpArg *args = argp;
-    HashMap *request = NULL;
-    HashMap *response = NULL;
-    
-    response = ASVerifyRequest(args);
-    if (response)
-    {
-        goto end;
-    }
-    if (HttpRequestMethodGet(args->ctx) != HTTP_POST)
-    {
-        HttpResponseStatus(args->ctx, HTTP_METHOD_NOT_ALLOWED);
-        response = MatrixCreateError(
-            "M_UNRECOGNIZED",
-            "Path /ping only accepts POST as a valid method."
-        );
-        goto end;
-    }
-    Log(LOG_INFO, "Pong!");
-
-    RequestJSON();
-
-    /* TODO: Load ping info */
-    response = HashMapCreate();
-end:
-    JsonFree(request);
-    return response;
-}

src/Routes/Transactions.c

@@ -30,7 +30,6 @@ RouteHead(RouteTxns, arr, argp)
 
     RequestJSON();
 
-    /* TODO: Do a thing with these. */
     events = JsonValueAsArray(HashMapGet(request, "events"));
     for (i = 0; i < ArraySize(events); i++)
     {

src/XMPP/Component.c

@@ -102,7 +102,6 @@ ComputeHandshake(char *shared, char *stream)
 bool
 XMPPAuthenticateCompStream(XMPPComponent *comp, char *shared)
 {
-    /* TODO */
     XMLexer *sax;
     XMLEvent *ev;
     char *stream_id, *handshake;
@@ -176,7 +175,7 @@ XMPPAuthenticateCompStream(XMPPComponent *comp, char *shared)
 
     ret = true;
     /* We can uhh, send stanzas, and receive them! */
-    Log(LOG_INFO, "Communications to '%s' established.", as);
+    Log(LOG_DEBUG, "Communications to '%s' established.", as);
 
     XMLFreeEvent(ev);
     Free(stream_id);

src/XMPPThread/Bridged.c

@@ -118,7 +118,6 @@ ParseeVerifyAllStanza(ParseeData *args, XMLElement *stanza)
     return ret;
 }
 
-/* TODO: Cache this information. */
 bool
 ServerHasXEP421(ParseeData *data, char *from)
 {

src/XMPPThread/PEP.c

@@ -7,6 +7,33 @@
 
 #include <pthread.h>
 
+#include <XMPP.h>
+#include <XML.h>
+
+XMLElement *
+CreatePubsubRequest(char *from, char *to, char *node)
+{
+    XMLElement *iq_req, *pubsub, *sub;
+    char *id;
+    iq_req = XMLCreateTag("iq");
+    XMLAddAttr(iq_req, "from", from);
+    XMLAddAttr(iq_req, "to", to);
+    XMLAddAttr(iq_req, "id", (id = StrRandom(16)));
+    XMLAddAttr(iq_req, "type", "set");
+
+    pubsub = XMLCreateTag("pubsub");
+    XMLAddAttr(pubsub, "xmlns", "http://jabber.org/protocol/pubsub");
+    XMLAddChild(iq_req, pubsub);
+
+    sub = XMLCreateTag("subscribe");
+    XMLAddAttr(sub, "node", node);
+    XMLAddAttr(sub, "jid", from);
+    XMLAddChild(pubsub, sub);
+
+    Free(id);
+    return iq_req;
+}
+
 struct PEPManager {
     pthread_mutex_t lock;
 

src/XMPPThread/Pubsub.c

@@ -1,32 +0,0 @@
-#include "XMPPThread/internal.h"
-
-#include <Cytoplasm/Memory.h>
-#include <Cytoplasm/Str.h>
-
-#include <XMPP.h>
-#include <XML.h>
-
-/* TODO: This should be in PEP.c */
-XMLElement *
-CreatePubsubRequest(char *from, char *to, char *node)
-{
-    XMLElement *iq_req, *pubsub, *sub;
-    char *id;
-    iq_req = XMLCreateTag("iq");
-    XMLAddAttr(iq_req, "from", from);
-    XMLAddAttr(iq_req, "to", to);
-    XMLAddAttr(iq_req, "id", (id = StrRandom(16)));
-    XMLAddAttr(iq_req, "type", "set");
-
-    pubsub = XMLCreateTag("pubsub");
-    XMLAddAttr(pubsub, "xmlns", "http://jabber.org/protocol/pubsub");
-    XMLAddChild(iq_req, pubsub);
-
-    sub = XMLCreateTag("subscribe");
-    XMLAddAttr(sub, "node", node);
-    XMLAddAttr(sub, "jid", from);
-    XMLAddChild(pubsub, sub);
-
-    Free(id);
-    return iq_req;
-}

src/XMPPThread/ReListener.c

@@ -58,6 +58,11 @@ XMPPDispatcher(void *argp)
             continue;
         }
 
+        if (args->verbosity >= PARSEE_VERBOSE_COMICAL)
+        {
+            Log(LOG_DEBUG, "Received stanza='%s'.", stanza->name);
+        } 
+
         if (StrEquals(stanza->name, "presence"))
         {
             PresenceStanza(args, stanza);
@@ -68,6 +73,10 @@ XMPPDispatcher(void *argp)
         {
             if (!MessageStanza(args, stanza, thread))
             {
+                if (args->verbosity >= PARSEE_VERBOSE_COMICAL)
+                {
+                    Log(LOG_DEBUG, "Done receiving message with an error");
+                } 
                 continue;
             }
         }
@@ -82,6 +91,11 @@ XMPPDispatcher(void *argp)
             StreamPrintf(StreamStdout(), "\n");
             StreamFlush(StreamStdout());
         }
+
+        if (args->verbosity >= PARSEE_VERBOSE_COMICAL)
+        {
+            Log(LOG_DEBUG, "Done receiving stanza='%s'.", stanza->name);
+        } 
         XMLFreeElement(stanza);
     }
 
@@ -180,9 +194,22 @@ ParseeXMPPThread(void *argp)
         stanza = XMLDecode(jabber->stream, false);
         if (!stanza)
         {
+            if (args->verbosity >= PARSEE_VERBOSE_COMICAL)
+            {
+                Log(LOG_DEBUG, "RECEIVED EOF.");
+            }
             break;
         }
 
+        if (args->verbosity >= PARSEE_VERBOSE_STANZA)
+        {
+            Stream *output = StreamStderr();
+            StreamPrintf(output,   "-------STANZA BEGIN-------"     "\n");
+            XMLEncode(output, stanza);
+            StreamPrintf(output, "\n--------STANZA END--------"     "\n");
+            StreamFlush(output);
+        }
+
         id = HashMapGet(stanza->attrs, "id");
         if (id)
         {
@@ -271,7 +298,7 @@ ParseeAwaitStanza(char *identifier, int64_t timeout)
     {
         return NULL;
     }
-    
+
     pthread_mutex_lock(&await_lock);
     if (HashMapGet(await_table, identifier))
     {

src/XMPPThread/Stanzas/IQ.c

@@ -120,7 +120,6 @@ IQResult(ParseeData *args, XMLElement *stanza, XMPPThread *thr)
 {
     XMLElement *vcard = XMLookForTKV(stanza, "vCard", "xmlns", "vcard-temp");
 
-
     /* TODO: Move this to PEP */
     XMLElement *event = XMLookForTKV(stanza, "pubsub", 
         "xmlns", "http://jabber.org/protocol/pubsub"
@@ -169,12 +168,16 @@ IQResult(ParseeData *args, XMLElement *stanza, XMPPThread *thr)
                     DbUnlock(args->db, avatars);
                     return;
                 }
+                JsonValueFree(JsonSet(
+                    json, JsonValueString(id),
+                    1, from
+                ));
+                DbUnlock(args->db, avatars);
 
                 base64 = TrimBase64(data->data);
                 b64len = base64 ? strlen(base64) : 0;
                 length = Base64DecodedSize(base64, b64len);
 
-                /* TODO: Bound checks for a size limit. */
                 bdata = (char *) Base64Decode(base64, b64len);
                 datastream = StrStreamReaderN(bdata, length);
 
@@ -185,12 +188,6 @@ IQResult(ParseeData *args, XMLElement *stanza, XMPPThread *thr)
                 from_matrix = ParseeGetBridgedUser(args, stanza);
                 ASSetAvatar(args->config, from_matrix, mxc);
 
-                JsonValueFree(JsonSet(
-                    json, JsonValueString(id),
-                    1, from
-                ));
-                DbUnlock(args->db, avatars);
-
                 Free(mxc);
                 Free(jid);
                 Free(bdata);

src/XMPPThread/Stanzas/Presence.c

@@ -95,7 +95,6 @@ PresenceStanza(ParseeData *args, XMLElement *stanza)
         if (!real_matrix || *real_matrix != '@')
         {
             Free(real_matrix);
-            /*real_matrix = ParseeEncodeJID(args->config, decode_from, false);*/
             real_matrix = ParseeGetBridgedUser(args, stanza);
         }
 
@@ -109,8 +108,6 @@ PresenceStanza(ParseeData *args, XMLElement *stanza)
         }
         else if (StrEquals(affiliation, "member"))
         {
-            /* TODO: Reconsider setting the PL if a member, as there's no 
-             * clear reason to do this in some cases. */
             power_level = 0;
         }
         else if (StrEquals(affiliation, "outcast"))
@@ -139,7 +136,7 @@ PresenceStanza(ParseeData *args, XMLElement *stanza)
         }
 
         /* Set the user's PL
-         * TODO: Do NOT change the PL of *real* people nilly-willy.
+         * NOTE: Do NOT change the PL of *real* people nilly-willy.
          * In some scenarios, this is really bad behaviour. */
         if (room)
         {
@@ -151,7 +148,6 @@ PresenceStanza(ParseeData *args, XMLElement *stanza)
              * lacking any checking. (--gen) */
             if (StrEquals(parsee, matrix_user_pl))
             {
-                /* TODO: Give the user an achievement, this is goofy. */
                 Log(LOG_ERR, "BAD PL DOWNGRADE (%d->%d)", level, power_level);
             }
             if (users && level != power_level && 
@@ -239,6 +235,13 @@ PresenceStanza(ParseeData *args, XMLElement *stanza)
         {
             return;
         }
+        if (args->verbosity >= PARSEE_VERBOSE_COMICAL)
+        {
+            Log(LOG_DEBUG, 
+                "Looking at VCard in presence from %s", 
+                oid
+            );
+        }
 
         avatars = DbLock(args->db, 1, "avatars");
         if (!avatars)
@@ -246,18 +249,32 @@ PresenceStanza(ParseeData *args, XMLElement *stanza)
             avatars = DbCreate(args->db, 1, "avatars");
         }
         json = DbJson(avatars);
+        if (args->verbosity >= PARSEE_VERBOSE_COMICAL)
+        {
+            Log(LOG_DEBUG, "VCard: Locked the database!");
+        }
 
         avatar_id = GrabString(json, 1, oid);
         if (avatar_id && StrEquals(avatar_id, p_dat->data))
         {
+            if (args->verbosity >= PARSEE_VERBOSE_COMICAL)
+            {
+                Log(LOG_DEBUG, 
+                    "VCard: %s is already cached for %s", avatar_id, oid
+                );
+            }
             DbUnlock(args->db, avatars);
             return;
         }
+
+        if (args->verbosity >= PARSEE_VERBOSE_COMICAL)
+        {
+            Log(LOG_DEBUG, "VCard: %s for %s", p_dat->data, oid);
+        }
         JsonValueFree(JsonSet(
             json, JsonValueString(p_dat->data),
             1, oid)
         );
-
         DbUnlock(args->db, avatars);
 
         from = ParseeJID(args);
@@ -273,8 +290,6 @@ PresenceStanza(ParseeData *args, XMLElement *stanza)
         XMLFreeElement(vcard_request);
         Free(from);
     }
-
-    /* TODO: Sending VCard on presence is slow and stalls the thread */
 #undef MUC_USER_NS
 }
 

src/include/Parsee.h

@@ -17,6 +17,11 @@
 #include <Command.h>
 #include <XMPP.h>
 
+#define PARSEE_VERBOSE_NONE     0
+#define PARSEE_VERBOSE_LOG      1
+#define PARSEE_VERBOSE_STANZA   2
+#define PARSEE_VERBOSE_COMICAL  53 /* MINUTES */
+
 typedef struct ParseeConfig {
     /* -------- MATRIX -------- */
     char    *as_token, *hs_token;
@@ -56,6 +61,8 @@ typedef struct ParseeData {
 
     Db *db;
     char *id;
+
+    int verbosity;
 } ParseeData;
 
 typedef struct Argument {

src/include/Routes.h

@@ -72,7 +72,6 @@ typedef struct ParseeCmdArg {
 /* A list of all routes. */
 #define ROUTES      X_ROUTE("/", RouteRoot) \
                     X_ROUTE("/_matrix/app/v1/transactions/(.*)", RouteTxns) \
-                    X_ROUTE("/_matrix/app/v1/ping", RoutePing) \
                     X_ROUTE("/_matrix/app/v1/users/(.*)", RouteUserAck) \
                     X_ROUTE("/_matrix/app/v1/rooms/(.*)", RouteRoomAck) \
                     X_ROUTE("/_matrix/client/v1/media/download/(.*)/(.*)", RouteMedia)

tools/noavatars.c

@@ -0,0 +1,39 @@
+/* noavatars.c - Clean up the avatar cache
+ * ============================================================
+ * Under CC0, as its a rather useful example of a Parsee tool.
+ * See LICENSE for more information about Parsee's licensing. */
+
+#include <Cytoplasm/Memory.h>
+#include <Cytoplasm/Json.h>
+#include <Cytoplasm/Log.h>
+#include <Cytoplasm/Db.h>
+
+#include <stdlib.h>
+
+int
+Main(Array *args, HashMap *env)
+{
+    char *db_path, *exec;
+    Db *parsee;
+
+    exec = ArrayGet(args, 0);
+
+    if (ArraySize(args) < 2)
+    {
+        Log(LOG_ERR, "Usage: %s [DB path]", exec);
+        return EXIT_FAILURE;
+    }
+
+    db_path = ArrayGet(args, 1);
+
+    parsee = DbOpenLMDB(db_path, 8 * 1024 * 1024);
+    if (parsee)
+    {
+        DbDelete(parsee, 1, "avatars");
+        DbClose(parsee);
+        return EXIT_SUCCESS;
+    }
+
+    Log(LOG_ERR, "%s: couldn't open DB '%s'", exec, db_path);
+    return EXIT_FAILURE;
+}