Added Scanner identification and MQTT handling

This commit is contained in:
Dirk Jahnke 2022-04-13 15:43:20 +02:00
parent 4da41fe1cd
commit 4b7234fa68
18 changed files with 595 additions and 76 deletions

View File

@ -1,3 +1,4 @@
from django.contrib import admin
from .models import Scanner
# Register your models here.
admin.site.register(Scanner)

View File

@ -0,0 +1,54 @@
from channels.generic.websocket import AsyncWebsocketConsumer
import json
import logging
from homelog import settings
LOGGER = logging.getLogger(__name__)
class BookerFeConsumer(AsyncWebsocketConsumer):
async def connect(self):
self.scanner_id = self.scope['url_route']['kwargs']['scanner_id']
self.scanner_group_name = self.scanner_id
self.group_name = self.scanner_group_name
#self.group_name = settings.MQTT_CHANNEL_NAME
# join to group
await self.channel_layer.group_add(self.scanner_group_name, self.channel_name)
topic = f"barcodescanner/{self.scanner_id}/scan"
LOGGER.debug(f"send mqtt_subscribe topic {topic} to channel group {self.scanner_group_name} (my channel is {self.channel_name})")
await self.channel_layer.send(settings.MQTT_CHANNEL_NAME, {
"type": "mqtt.subscribe",
"topic": topic,
"group": self.scanner_group_name
})
await self.accept()
async def disconnect(self):
# leave group
await self.channel_layer.group_discard(self.group_name, self.channel_name)
LOGGER.debug(f"disconnect from channel group {self.group_name}")
async def receive(self, text_data):
print('>>>', text_data)
msg = '{"message":"pong!"}' if text_data == '{"message":"ping"}' else text_data
print('<<<', msg)
await self.channel_layer.group_send(
self.scanner_group_name,
{
'type': 'distribute',
'text': msg
}
)
async def distribute(self, event):
valother = event['text']
LOGGER.debug(f"distribute text={valother}")
await self.send(text_data=valother)
async def mqtt_message(self, event):
message = event['message']
topic = message['topic']
payload = message['payload']
LOGGER.debug(f"Received message with payload={payload}, topic={topic}")
await self.send(text_data=json.dumps({'message': payload, 'topic': topic}))

View File

@ -1,3 +1,18 @@
from django.db import models
from django.conf import settings
from homelog.system_user import get_deleted_user
# Create your models here.
class Scanner(models.Model):
named_id = models.CharField(max_length=40, unique=True)
description = models.CharField(max_length=200)
lwt_topic = models.CharField(max_length=200)
last_online_ts = models.DateTimeField('last datetime this scanner has been online')
assigned_channel_name = models.CharField(max_length=200)
assigned_group_name = models.CharField(max_length=200)
created_ts = models.DateTimeField('datetime created', auto_now_add=True)
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET(get_deleted_user),
related_name='created_scanners')
changed_ts = models.DateTimeField('datetime updated', auto_now=True)
changed_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET(get_deleted_user),
related_name='changed_scanners')

View File

@ -1,48 +0,0 @@
from channels.generic.websocket import AsyncWebsocketConsumer
import json
'''
class ScannerCollector(WebsocketConsumer):
def connect(self):
self.accept()
def disconnect(self, close_code):
pass
def receive(self, text_data):
text_data_json = json.loads(text_data)
message = text_data_json['message']
self.send(text_data=json.dumps({
'message': message
}))
'''
class ScannerCollector(AsyncWebsocketConsumer):
async def connect(self):
self.group_name = 'scanner'
# join to group
await self.channel_layer.group_add(self.group_name, self.channel_name)
await self.accept()
async def disconnect(self):
# leave group
await self.channel_layer.group_discard(self.group_name, self.channel_name)
async def receive(self, text_data):
print('>>>', text_data)
msg = '{"message":"pong!"}' if text_data == '{"message":"ping"}' else text_data
print('<<<', msg)
await self.channel_layer.group_send(
self.group_name,
{
'type': 'distribute',
'text': msg
}
)
async def distribute(self, event):
valother = event['text']
await self.send(text_data=valother)

View File

@ -0,0 +1,77 @@
import datetime
import logging
import re
import json
from channels.generic.websocket import AsyncWebsocketConsumer
from booker.models import Scanner
from asgiref.sync import sync_to_async
from channels.db import database_sync_to_async
from django.contrib.auth import get_user_model
from homelog.system_user import get_system_user
from homelog import settings
from django.core.exceptions import ObjectDoesNotExist
LOGGER = logging.getLogger(__name__)
class ScannerAnnounceConsumer(AsyncWebsocketConsumer):
async def connect(self):
self.group_name = 'scanner-announce'
# join to group
await self.channel_layer.group_add(self.group_name, self.channel_name)
LOGGER.debug(f"send mqtt_subscribe to channel layer {settings.MQTT_CHANNEL_NAME} to answer on channel {self.group_name} (my channel is {self.channel_name})")
await self.channel_layer.send(settings.MQTT_CHANNEL_NAME, {
"type": "mqtt.subscribe",
"topic": "barcodescanner/+/status",
"group": self.group_name
})
LOGGER.debug("ScannerAnnounceConsumer initialized")
await self.accept()
async def receive(self, mqtt_message):
pass
async def disconnect(self, code):
await self.channel_layer.send(settings.MQTT_CHANNEL_NAME, {
"type": "mqtt.unsubscribe",
"topic": "barcodescanner/#/status",
"group": self.group_name
})
LOGGER.debug(f"Disconnect from scanner-announce, unsubscribing topic; code={code}")
await self.channel_layer.group_discard(self.group_name, self.channel_name)
async def distribute(self, event):
valother = event['text']
await self.send(text_data=valother)
@database_sync_to_async
def get_or_create_scanner(self, named_id, topic):
scanner, created = Scanner.objects.get_or_create(named_id=named_id, defaults={
'lwt_topic': topic,
'last_online_ts': datetime.datetime.now(),
'created_by': get_system_user(),
'changed_by': get_system_user()
})
if created:
LOGGER.debug(f"Created new scanner entry for {named_id}")
else:
LOGGER.debug(f"Updated scanner entry for {named_id}")
return scanner
async def mqtt_message(self, event):
message = event['message']
topic = message['topic']
qos = message['qos']
payload = message['payload']
print('Received a message at topic: ', topic)
print('with payload ', payload)
print('and QOS ', qos)
named_id = 'invalid'
m = re.match(r'barcodescanner\/(barcodescan-[0-9,a-f]{6})\/status', topic)
if m:
named_id = m[1]
await self.send(text_data=json.dumps({'message': payload, 'scanner': named_id, 'topic': topic}))
LOGGER.debug(f"Got MQTT message from {topic} with payload {payload}")
if payload == 'Online' or payload == 'online':
scanner = await self.get_or_create_scanner(named_id=named_id, topic=topic)

View File

@ -4,21 +4,55 @@
{% block title %}Scanner Index{% endblock %}
{% block content %}
What chat room would you like to enter?<br>
<input id="room-name-input" type="text" size="100"><br>
<input id="room-name-submit" type="button" value="Enter">
<div class="container">
<div class="row">
<h1>Booker</h1>
<p>Choose a scanner you want to use to book assets and container into/outof containers.</p>
</div>
<div class="row">
<div class="cols-12">
<textarea id="chat-log" cols="100" rows="7"></textarea><br>
<script>
document.querySelector('#room-name-input').focus();
document.querySelector('#room-name-input').onkeyup = function(e) {
if (e.keyCode === 13) { // enter, return
document.querySelector('#room-name-submit').click();
}
};
What scanner would you like to use? (Refresh page if your new scanner is not shown)<br>
{% for scanner in object_list %}
<a href="/booker/{{ scanner.named_id }}/">{{ scanner.named_id }}: {{ scanner.description }}</a><br />
{% endfor %}
<input id="scanner-name-input" type="text" size="100"><br>
<input id="scanner-name-submit" type="button" value="Enter">
document.querySelector('#room-name-submit').onclick = function(e) {
var roomName = document.querySelector('#room-name-input').value;
window.location.pathname = '/booker/' + roomName + '/';
};
</script>
<script>
document.querySelector('#scanner-name-input').focus();
document.querySelector('#scanner-name-input').onkeyup = function(e) {
if (e.keyCode === 13) { // enter, return
document.querySelector('#scanner-name-submit').click();
}
};
document.querySelector('#scanner-name-submit').onclick = function(e) {
var scannerName = document.querySelector('#scanner-name-input').value;
window.location.pathname = '/booker/' + scannerName + '/';
};
</script>
<script>
const chatSocket = new WebSocket(
'ws://'
+ window.location.host
+ '/ws/scanner-announce/'
);
chatSocket.onmessage = function(e) {
const data = JSON.parse(e.data);
document.querySelector('#chat-log').value += (data.scanner + ': ' + data.message + '\n');
};
chatSocket.onclose = function(e) {
console.error('Chat socket closed unexpectedly');
document.querySelector('#chat-log').value += ('ERROR: Chat socket closed unexpectedly!\n');
};
</script>
</div>
</div>
</div>
{% endblock %}

View File

@ -7,15 +7,15 @@
<textarea id="chat-log" cols="100" rows="20"></textarea><br>
<input id="chat-message-input" type="text" size="100"><br>
<input id="chat-message-submit" type="button" value="Send">
{{ scanner_id|json_script:"room-name" }}
{{ scanner_id|json_script:"scanner_id" }}
<script>
const roomName = JSON.parse(document.getElementById('room-name').textContent);
const scannerId = JSON.parse(document.getElementById('scanner_id').textContent);
const chatSocket = new WebSocket(
'ws://'
+ window.location.host
+ '/ws/scannerdata/'
+ roomName
+ '/ws/scanner/'
+ scannerId
+ '/'
);

View File

@ -4,6 +4,6 @@ from . import views
app_name = 'booker'
urlpatterns = [
path('', views.index, name='index'),
path('', views.IndexView.as_view(), name='index'),
path('<str:scanner_id>/', views.scanner, name='scanner')
]

View File

@ -1,10 +1,14 @@
from django.shortcuts import render
from django.views.generic import ListView
from .models import Scanner
def index(request):
return render(request, 'booker/index.html')
class IndexView(ListView):
model = Scanner
template_name = 'booker/index.html'
def scanner(request, scanner_id):
return render(request, 'booker/scanner.html', {
'scanner_id': scanner_id
})
})

5
docker/create_redis.sh Normal file
View File

@ -0,0 +1,5 @@
#!/bin/sh
docker run --name redis_django \
-p 6379:6379/tcp \
-e ALLOW_EMPTY_PASSWORD=yes \
-d bitnami/redis:latest

2
docker/start_redis.sh Normal file
View File

@ -0,0 +1,2 @@
#!/bin/sh
docker start redis_django

3
docker/stop_redis.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/sh
docker stop redis_django

View File

@ -5,6 +5,8 @@ from channels.routing import ProtocolTypeRouter, URLRouter
from django.core.asgi import get_asgi_application
import homelog.routing
from channels.routing import ChannelNameRouter
from .mqtt_consumer import MqttConsumer
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'homelog.settings')
@ -14,4 +16,7 @@ django_application = get_asgi_application()
application = ProtocolTypeRouter({
"http": get_asgi_application(),
"websocket": AuthMiddlewareStack(URLRouter(homelog.routing.websocket_urlpatterns)),
"channel": ChannelNameRouter({
"mqtt": MqttConsumer.as_asgi(),
})
})

View File

@ -0,0 +1,252 @@
import asyncio
import functools
import json
import logging
import os
import signal
import socket
from gmqtt import Client as MQTTClient
from gmqtt.mqtt.constants import MQTTv311, MQTTv50
LOGGER = logging.getLogger(__name__)
class ChannelsMQTTProxy:
def __init__(self, channel_layer, settings):
self.channel_layer = channel_layer
# MQTTClient takes an identifier which is seen at the broker
# Creating the client does not connect.
self.mqtt = MQTTClient(
f"ChannelsMQTTProxy@{socket.gethostname()}.{os.getpid()}")
self.mqtt.set_auth_credentials(username=settings.MQTT_USER,
password=settings.MQTT_PASSWORD)
self.mqtt_host = settings.MQTT_HOST
try:
self.mqtt_version = settings.MQTT_VERSION
except AttributeError:
self.mqtt_version = None
# Hook up the callbacks and some lifecycle management events
self.mqtt.on_connect = self._on_connect
self.mqtt.on_disconnect = self._on_disconnect
self.mqtt.on_subscribe = self._on_subscribe
self.mqtt.on_message = self._on_message
self.stop_event = asyncio.Event()
self.connected = asyncio.Event()
try:
self.mqtt_channel_name = settings.MQTT_CHANNEL_NAME
except AttributeError:
self.mqtt_channel_name = "mqtt"
self.mqtt_channel_publish = f"{self.mqtt_channel_name}.publish"
self.mqtt_channel_message = f"{self.mqtt_channel_name}.message"
self.subscriptions = {}
async def run(self):
"""This connects to the mqtt broker (retrying forever), then calls the
overrideable :func:`setup()` method finally awaits
:func:`ask_exit` is called at which point it exits cleanly.
Alternatively you can call :func:`connect()` and then wait for
:func:`finish()` Once connected the underlying qmqtt client
will re-connect if the connection is lost.
"""
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGINT, self.ask_exit)
loop.add_signal_handler(signal.SIGTERM, self.ask_exit)
await self.connect()
await self.finish()
async def connect(self):
# Connect to the broker
if self.mqtt_version == 311:
version = MQTTv311
else:
version = MQTTv50
while not self.mqtt.is_connected:
try:
await self.mqtt.connect(self.mqtt_host, version=version)
except Exception as e:
LOGGER.warning(f"Error trying to connect: {e}. Retrying.")
await asyncio.sleep(1)
self.connected.set()
async def finish(self):
# This will wait until the client is signalled
LOGGER.debug("Waiting for stop event")
await self.stop_event.wait()
await self.mqtt.disconnect()
LOGGER.debug("MQTT client disconnected")
def _on_connect(self, _client, _flags, _rc, _properties):
for s in self.subscriptions.keys():
LOGGER.debug(f"Re-subscribing to {s}")
self.mqtt.subscribe(s)
LOGGER.debug('Connected and subscribed')
def _on_disconnect(self, _client, _packet, _exc=None):
LOGGER.debug('Disconnected')
async def _on_message(self, _client, topic, payload, qos, properties):
LOGGER.debug(f"{topic} => '{payload}' props:{properties}")
# Check properties for 'retain' (which in this context means
# the message is being sent from retained backing store) and
# drop those.
# Eventually find a way to direct these to support code which can build
# initial state and keep it up-to-date? This is linked to
# connect-on-startup.
if properties["retain"] == 1:
LOGGER.debug(f"Dropping replayed retained message")
return
# Compose a Channel message
payload = payload.decode("utf-8")
try:
payload = json.loads(payload)
except:
LOGGER.debug("Payload is not JSON - sending it raw")
pass
msg = {
"topic": topic,
"payload": payload,
"qos": qos,
}
event = {
"type": self.mqtt_channel_message, # default "mqtt.message"
"message": msg
}
tasks = list()
for grp in self.groups_matching_topic(topic):
tasks.append(self.channel_layer.group_send(grp, event))
LOGGER.debug(f"Calling {grp} handler for {topic}")
try:
await asyncio.gather(*tasks)
except Exception as e:
LOGGER.error("Cannot send event: {event}")
LOGGER.exception(e)
def subscribe(self, topic, group):
"""Subscribes a group to an MQTT topic (passed directly to MQTT)"""
if topic not in self.subscriptions:
LOGGER.debug(f"New subscription for {topic}")
self.subscriptions[topic] = []
# We need to mqtt-subscribe now:
# This actually just sends a subscribe packet. We should
# store this and the details and handle the setup in
# _on_subscribe callback when the _mid (message id) is
# confirmed.
_mid = self.mqtt.subscribe(topic)
else:
if group in self.subscriptions[topic]:
LOGGER.debug(f"{group} already subscibed to {topic}")
return
LOGGER.debug(f"{group} subscribed to {topic}")
self.subscriptions[topic].append(group)
def _on_subscribe(self, _client, _mid, _qos, _properties):
LOGGER.debug('Subscribe callback {_mid}')
async def unsubscribe(self, topic, group):
"""Un subscribes a group to an MQTT topic"""
LOGGER.debug(f"unsubscribe {group} from {topic}")
if topic in self.subscriptions:
groups = self.subscriptions[topic]
if group in groups:
LOGGER.debug(f"{group} being unsubscribed from {topic}")
groups.delete(topic)
else:
LOGGER.debug(f"{group} not subscribed to {topic}")
if not len(groups):
LOGGER.debug(f"No more {group}s, unsubscribing to {topic}")
self.mqtt.unsubscribe(topic)
LOGGER.debug(f"{topic} not subscribed")
def publish(self, topic=None, payload=None, qos=2, retain=True):
"""Publish :param payload: to :param topic:"""
LOGGER.debug(f"Publishing {topic} = {payload}")
self.mqtt.publish(topic, payload, qos=qos, retain=retain)
def ask_exit(self):
"""Handle outstanding messages and cleanly disconnect"""
LOGGER.warning(f"{self} received signal asking to exit")
self.stop_event.set()
def groups_matching_topic(self, topic):
groups = set()
for sub, gs in self.subscriptions.items():
if sub == topic: # simple match
groups.update(gs)
elif self.topic_matches_sub(sub, topic):
LOGGER.debug(f"Found matching groups {gs}")
groups.update(gs)
return groups
# Taken from paho-mqtt - thanks :)
@staticmethod
def topic_matches_sub(sub, topic):
"""Check whether a topic matches a subscription.
For example:
foo/bar would match the subscription foo/# or +/bar
non/matching would not match the subscription non/+/+
"""
result = True
multilevel_wildcard = False
slen = len(sub)
tlen = len(topic)
if slen > 0 and tlen > 0:
if (sub[0] == '$' and topic[0] != '$') or (topic[0] == '$' and sub[0] != '$'):
return False
spos = 0
tpos = 0
while spos < slen and tpos < tlen:
if sub[spos] == topic[tpos]:
if tpos == tlen-1:
# Check for e.g. foo matching foo/#
if spos == slen-3 and sub[spos+1] == '/' and sub[spos+2] == '#':
result = True
multilevel_wildcard = True
break
spos += 1
tpos += 1
if tpos == tlen and spos == slen-1 and sub[spos] == '+':
spos += 1
result = True
break
else:
if sub[spos] == '+':
spos += 1
while tpos < tlen and topic[tpos] != '/':
tpos += 1
if tpos == tlen and spos == slen:
result = True
break
elif sub[spos] == '#':
multilevel_wildcard = True
if spos+1 != slen:
result = False
break
else:
result = True
break
else:
result = False
break
if not multilevel_wildcard and (tpos < tlen or spos < slen):
result = False
return result

View File

@ -0,0 +1,33 @@
[
{
"model": "auth.user",
"pk": 1,
"fields": {
"password": "pbkdf2_sha256$320000$fRhSLDGgXfVXoSilNbJhj4$2zSGFf6j9IkPdBRiosaUO6ctEZl2FruRv/LnvxK5gt8=",
"last_login": "2022-04-02T17:29:25.538Z",
"is_superuser": true,
"username": "dirk",
"first_name": "",
"last_name": "",
"email": "dirk.jahnke@mailbox.org",
"is_staff": true, "is_active": true, "date_joined": "2022-03-09T14:13:51.157Z", "groups": [], "user_permissions": []}
},
{
"model": "auth.user",
"pk": 2,
"fields": {
"password": "pbkdf2_sha256$320000$lP0jbaUwn7gH1bKcMKS8eH$gGDViFe8zdHWiA4bSzldaQvTZoe2wm0+scNgMju8txE=",
"last_login": null,
"is_superuser": false,
"username": "system",
"first_name": "Tech",
"last_name": "System",
"email": "",
"is_staff": false,
"is_active": true,
"date_joined": "2022-04-05T08:08:26Z",
"groups": [],
"user_permissions": []
}
}
]

55
homelog/mqtt_consumer.py Normal file
View File

@ -0,0 +1,55 @@
import asyncio
import logging
import sys
from channels.consumer import AsyncConsumer
from channels.layers import get_channel_layer
from django.conf import settings
from homelog.channelsmqttproxy import ChannelsMQTTProxy
LOGGER = logging.getLogger(__name__)
class MqttConsumer(AsyncConsumer):
"""The MqttConsumer is run as a channels worker. It starts an MQTT
client and handles subscription and message distribution via
Channels messages.
"""
def __init__(self):
super().__init__()
self.mqttproxy = ChannelsMQTTProxy(get_channel_layer(), settings)
self.task = asyncio.create_task(self.mqttproxy.run())
self.task.add_done_callback(self.finish)
LOGGER.debug("MqttConsumer initialized")
def finish(self, task):
# For some reason the Channels worker doesn't seem to exit
# if there's a task with a loop signal handler
sys.exit(0)
async def mqtt_subscribe(self, event):
"""This is the mqtt.subscribe channel message handler. It subscribes
a channel group to a topic. All messages received for that
topic will be sent to all members of the group using
'mqtt.message'. Multiple subscriptions are allowed. The topic
uses MQTT wildcard syntax. The same topic may be subscibed by
multiple channel groups
"""
LOGGER.debug(f"MqttConsumer.mqtt_subscribe topic={event['topic']}, group={event['group']}")
topic = event['topic']
group = event['group']
LOGGER.debug(f"subscribe to {topic} for {group}")
await self.mqttproxy.connected.wait()
self.mqttproxy.subscribe(topic, group)
async def mqtt_publish(self, event):
"""The event contains a publish dict which is used by the mqtt client.
The values : topic, payload, qos & retain may be specified.
"""
LOGGER.debug(event)
publish = event['publish']
# do something with topic and payload
LOGGER.debug(f"MQTT publish ({publish})")
await self.mqttproxy.connected.wait()
self.mqttproxy.publish(**publish)

View File

@ -1,12 +1,15 @@
from channels.routing import ProtocolTypeRouter, URLRouter
from channels.auth import AuthMiddlewareStack
from django.urls import path, re_path
from booker.mqtt_collector import ScannerCollector
from booker.booker_fe_consumer import BookerFeConsumer
from booker.scanner_announce_consumer import ScannerAnnounceConsumer
from django.core.asgi import get_asgi_application
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'homelog.settings')
websocket_urlpatterns = [
re_path(r'ws/scannerdata/(?P<scanner_id>\w+)/$', ScannerCollector.as_asgi()),
re_path(r'ws/scanner/(?P<scanner_id>[\w-]+)/$', BookerFeConsumer.as_asgi()),
re_path(r'ws/scanner-announce/$', ScannerAnnounceConsumer().as_asgi()),
]

View File

@ -77,7 +77,11 @@ ASGI_APPLICATION = 'homelog.asgi.application'
CHANNEL_LAYERS = {
'default': {
'BACKEND': 'channels.layers.InMemoryChannelLayer',
#'BACKEND': 'channels.layers.InMemoryChannelLayer',
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {
"hosts": [("speicher2.fritz.box", 6379)],
},
}
}
@ -142,3 +146,23 @@ MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
LOGIN_URL = '/accounts/login/'
MQTT_HOST = "hassio.fritz.box"
MQTT_USER = "djangoHomelog"
MQTT_PASSWORD = "NbuFz8RA7rgT3iPPqS"
MQTT_VERSION = 311
MQTT_CHANNEL_NAME = "mqtt"
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'class': 'logging.StreamHandler',
},
},
'root': {
'handlers': ['console'],
'level': 'DEBUG',
},
}