izaliamae/uncia
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request 'switch out sanic for custom server' (#1) from dev into main

Browse source 
Reviewed-on: #1
This commit is contained in:
Izalia Mae 2021-10-26 06:06:47 -04:00
parent c86bb215ec 9cf61e5895
commit b8f30d98ec
9 changed files with 240 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
requirements.txt
View file

@ -1,4 +1,3 @@
izzylib[hasher,http_server,http_signatures,http_urllib_client,sql,template] @ git+https://git.barkshark.xyz/izaliamae/izzylib@5e3f36af449dcc10cc28de09a8e755a252120f78
izzylib[hasher,http_server_async,http_signatures,http_urllib_client,sql,template] @ git+https://git.barkshark.xyz/izaliamae/izzylib@cc8f8b09a1c991b4328bf405c88dcf90626adeb7
pyyaml==5.4.1
apscheduler==3.8.0

2
uncia/__init__.py
View file

@ -1,4 +1,4 @@
__package__ = 'Uncia Relay'
__version__ = '0.1.0'
__version__ = '0.1.1'
__author__ = 'Zoey Mae'
__homepage__ = 'https://git.barkshark.xyz/izaliamae/uncia'

10
uncia/database/put.py
View file

@ -91,20 +91,24 @@ def cmd_instance(self, inbox, actor, followid=None):
def cmd_retry(self, inbox, data, headers={}, timestamp=None):
row = self.get.retry(data.id, inbox)
instance = self.get.instance(inbox)
row = instance.retry(data.id)
if not row:
instance = self.get.instance(inbox)
logging.verbose(f'Putting new retry in db for {instance.domain}: {data.id}')
row = self.insert('retry',
msgid = data.id,
inboxid = instanceid,
inboxid = instance.id,
data = data,
headers = headers,
timestamp = timestamp or datetime.now(),
return_row = True
)
else:
logging.verbose(f'Retry for {instance.domain} already in db: {data.id}')
return row

29
uncia/functions.py
View file

@ -1,7 +1,7 @@
import json
from functools import wraps
from izzylib import LruCache, logging
from izzylib import DotDict, LruCache, logging
from izzylib.http_urllib_client import HttpUrllibClient
from izzylib.http_urllib_client.error import MaxRetryError
@ -111,19 +111,28 @@ def push_message(inbox, message, headers={}):
keyid = f'https://{config.host}/actor#main-key'
)
except MaxRetryError:
return
except MaxRetryError as e:
response = DotDict(status=0, error=e)
except Exception as e:
logging.debug(f'push_message: {e.__class__.__name__}: {e}')
return
#logging.debug(f'push_message: {e.__class__.__name__}: {e}')
response = DotDict(status=0, error=e)
if response.status not in [200, 202]:
try:
body = response.dict
except:
body = response.text
s.put.retry(inbox, message, headers)
logging.debug(f'Error from {inbox}: {body}')
if response.status:
try:
body = response.dict
except:
body = response.text
logging.debug(f'Error from {inbox}: {response.status} {body}')
else:
logging.debug(f'Error from {inbox}: {response.error}')
return
logging.debug(f'Pushed message to {inbox}:', message['id'])
return response

113
uncia/manage.py
View file

@ -1,9 +1,11 @@
import json, sys
import json, os, sys
from configparser import ConfigParser
from datetime import datetime
from envbash import load_envbash
from getpass import getuser
from izzylib import DotDict, Path, boolean, logging, prompt
from grp import getgrgid
from izzylib import DotDict, Path, boolean, logging, prompt, sudo
from izzylib.sql import Database
from os import environ as env
from urllib.parse import urlparse
@ -13,6 +15,7 @@ from .config import config, dbconfig, path, write_config
from .database import db
from .functions import fetch_actor, get_inbox
from .messages import Message
from .server import retry_instance
exe = f'{sys.executable} -m uncia.manage'
@ -50,6 +53,13 @@ class Command:
python3 -m uncia.manage help:
Show this message.
python3 -m uncia.manage setup:
A series of questions will be asked to generate the env file.
python3 -m uncia.manage install [systemd or sysv]:
Generate an init service config and put it in the right location. Uses
"systemd" by default, but "sysv" can be specified instead
python3 -m uncia.manage list:
List currently subscribed instances.
@ -60,14 +70,14 @@ python3 -m remove <actor, inbox, or domain>:
Remove an instance and any retries associated with it from the database.
python3 -m uncia.manage ban <handle@domain or domain> [*reason]:
Ban a user or domain. A reason may optionally be specified.
Ban a user or domain. A reason may optionally be specified. Quotes not necessary for reason.
python3 -m uncia.manage unban <handle@domain or domain>: **
Unban a user or domain
python3 -m uncia.manage bans ["users" or "domains"]:
List the currently banned domains and users. A ban type ("users"/"domains")
may be specified to only list that type.
List the currently banned domains and users. A ban type ("users"/"domains")
may be specified to only list that type.
python3 -m uncia.manage rules [list]:
List the current rules.
@ -95,15 +105,77 @@ python3 -m uncia.manage accept <actor, inbox, or domain>: *
python3 -m uncia.manage deny <actor, inbox, or domain>: *
Reject a request.
python3 -m uncia.manage retries [domain]:
List all of the current retires. If a domain is specified, only list
retries from that instance
python3 -m uncia.manage retry [domain]:
Re-run retries. If a domain is specified, only the retries from that
instance will be ran.
python3 -m uncia.manage convert [pleroma or uncia]:
Convert the database and config of another relay. Tries to convert a
Pleroma Relay by default.
* = The relay needs to be running for the command to work
** = Caching prevents this from taking effect until restart.
** = Caching prevents this from taking effect until restart
'''
def cmd_setup(self):
pass
def cmd_install(self, init='systemd'):
if init == 'systemd':
srvfile = path.config.join('uncia.service')
initcfg = ConfigParser()
initcfg['Unit'] = {
'Description': 'Uncia Relay',
'After': 'network.target'
}
initcfg['Service'] = {
'User': getuser(),
'Group': getgrgid(os.getgid()),
'WorkingDirectory': Path.cwd,
'ExecStart': f'{sys.executable} -m uncia',
'Restart': 'always',
'StartLimitIntervalSec': 30,
'StartLimitBurst': 5
}
initcfg['Install'] = {
'WantedBy': 'multi-user.target'
}
with srvfile.open('w') as fd:
initcfg.write(fd)
elif init == 'sysv':
return '¯\_(ツ)_/¯'
else:
return f'Error: Invalid service type: {init}\nValid types: systemd, sysv'
return f'Service file saved to {path.config.join("uncia.service")}'
# incomplete code to copy service file
#copy_srv = None
#while type(copy_srv) != bool:
#if copy_srv != None:
#print('Please specify a boolean (yes/no, y/n, true/false, etc)')
#copy_srv = prompt('Copy service file to /etc/systemd/system/uncia.service?',
#valtype = lambda x: boolean(x, return_value=True),
#default = True
#)
def cmd_config(self, key=None, value=None):
with db.session as s:
if key and value:
@ -309,6 +381,35 @@ python3 -m uncia.manage convert [pleroma or uncia]:
return data or 'No banned domains or users yet'
def cmd_retries(self, domain=None):
with db.session as s:
if domain:
instances = [s.get.instance(domain)]
else:
instances = s.search('inbox')
retries = []
for instance in instances:
for retry in instance.retries():
retries.append(f'{retry.timestamp} {retry.msgid}')
return '\n'.join(retries)
def cmd_retry(self, domain=None):
with db.session as s:
if domain:
instances = [s.get.instance(domain)]
else:
instances = s.search('inbox')
for instance in instances:
retry_instance(instance)
def cmd_convert(self, relay='uncia'):
if relay.lower() == 'uncia':
return self.convert_uncia()

89
uncia/middleware.py
View file

@ -1,7 +1,6 @@
from izzylib import logging
from izzylib.http_server import MiddlewareBase
from izzylib.http_server_async import error
from izzylib.http_signatures import parse_signature, verify_request, verify_headers
from izzylib.sql.rows import Row
from .database import db
@ -53,58 +52,60 @@ def ban_check(s, request):
return True
class AuthCheck(MiddlewareBase):
attach = 'request'
async def AuthCheck(request):
validated = False
token = request.headers.getone('token')
with db.session as s:
request.token = s.fetch('token', code=token)
request.user = s.fetch('user', id=request.token.id) if request.token else None
request.signature = parse_signature(request.headers.getone('signature'))
request.instance = None
request.actor = None
async def handler(self, request, response):
validated = False
token = request.headers.get('token')
if request.signature:
request.instance = s.get.instance(request.signature.domain)
request.actor = fetch_actor(request.signature.actor)
with db.session as s:
request.token = s.fetch('token', code=token)
request.user = s.fetch('user', id=request.token.id) if request.token else None
request.signature = parse_signature(request.headers.get('signature'))
request.instance = None
request.actor = None
if request.signature.top_domain in blocked_instances:
raise error.Teapot('This teapot kills fascists')
if request.signature:
request.instance = s.get.instance(request.signature.domain)
request.actor = fetch_actor(request.signature.actor)
if ban_check(s, request):
raise error.Forbidden('no')
if request.signature.top_domain in blocked_instances:
return response.text(f'This teapot kills fascists', status=418)
if request.path in ['/inbox', '/actor'] and request.method.lower() == 'post':
## The actor was deleted, so return a 200 to not get the same message over and over again
if request.signature and request.actor == False:
raise error.Ok({'error': 'Could not fetch deleted actor'})
if ban_check(s, request):
return response.text('no', status=403)
if not request.actor:
raise error.Unauthorized({'error': 'Could not get actor'})
if request.path in ['/inbox', '/actor'] and request.method.lower() == 'post':
## The actor was deleted, so return a 202 to not get the same message over and over again
if request.actor == False:
return response.json({'error': 'Could not fetch deleted actor'}, status=202)
try:
data = await request.json()
if not request.actor:
return response.json({'error': 'Could not get actor'}, status=401)
except:
logging.verbose('Failed to parse post data')
raise error.BadRequest({'error': 'Invalid data'})
try:
data = request.data.json
if type(request.actor).__name__ == 'Row':
logging.warning('Actor data is a db row:', request.actor)
raise error.InternalServerError({'error': f'An unknown error happened'})
except:
logging.verbose('Failed to parse post data')
return response.json({'error': f'Invalid data'}, status=400)
if not request.instance and data.get('type', '').lower() != 'follow':
raise error.Unauthorized({'error': f'Follow the relay first'})
if type(request.actor).__name__ == 'Row':
logging.warning('Actor data is a db row:', request.actor)
return response.error({'error': f'An unknown error happened'}, status=500)
#validated = verify_request(request, request.actor)
validated = verify_headers(
request.headers.as_dict(),
request.method,
request.path,
request.actor,
await request.body())
if not request.instance and data.get('type', '').lower() != 'follow':
return response.json({'error': f'Follow the relay first'}, status=401)
if not validated:
logging.debug(f'Not validated: {request.signature.actor}')
raise error.Unauthorized({'error': f'Failed signature check'})
validated = verify_request(request, request.actor)
if not validated:
logging.debug(f'Not validated: {request.signature.actor}')
return response.json({'error': f'Failed signature check'}, status=401)
if any(map(request.path.startswith, auth_paths)) and not request.user:
return response.redir('/login')
if any(map(request.path.startswith, auth_paths)) and not request.user:
raise error.Found('/login')

2
uncia/processing.py
View file

@ -126,7 +126,7 @@ class ProcessData:
traceback.print_exc()
if not response or response.status not in [200, 202]:
logging.verbose(f'Failed to send object announce to {instance.domain}: {object.id}')
logging.verbose(f'Failed to send announce object to {instance.domain}: {object.id}')
s.put.retry(instance.inbox, msg)
if response:

43
uncia/server.py
View file

@ -1,10 +1,12 @@
from apscheduler.schedulers.background import BackgroundScheduler
import asyncio
from izzylib import logging
from izzylib.http_server import Application, Request
from izzylib.http_server_async import Application
from . import __version__, views
from .config import config, path, first_start
from .database import db
from .functions import push_message
from .middleware import AuthCheck
@ -25,35 +27,48 @@ with db.session as s:
listen = config.listen,
port = config.port,
host = config.host,
workers = config.workers,
git_repo = 'https://git.barkshark.xyz/izaliamae/uncia',
proto = 'https',
tpl_search = [path.frontend],
tpl_context = template_context,
class_views = [getattr(views, view) for view in dir(views) if view.startswith('Uncia')]
views = [getattr(views, view) for view in dir(views) if view.startswith('Uncia')],
middleware = [AuthCheck]
)
app.add_middleware(AuthCheck)
app.static('/style', path.frontend.join('style'))
app.add_static('/style', path.frontend.join('style'))
def run_retries():
async def run_retries():
await asyncio.sleep(60 * 60)
with db.session as s:
for instance in s.get.instance_list():
for instance in s.search('inbox'):
retry_instance(instance)
def retry_instance(instance):
retries = instance.retries()
fails = 0
if not retries:
return
logging.debug(f'Retrying {len(retries)} message(s) for {instance.domain}')
logging.verbose(f'Retrying {len(retries)} message(s) for {instance.domain}')
for retry in instance.retries():
try:
push_message(instance.inbox, retry.data, headers=retry.headers)
if push_message(instance.inbox, retry.data, headers=retry.headers):
fails = 0
with db.session as s:
s.remove('retry', row=retry)
else:
fails += 1
if fails >= 5:
logging.verbose(f'Failed 5 times in a row when retrying messages for {instance.domain}')
return
except Exception as e:
logging.debug(f'{e.__class__.__name__}: {e}')
@ -65,10 +80,4 @@ def main():
logging.error(f'Uncia has not been configured yet. Please edit {config.envfile} first.')
return
scheduler = BackgroundScheduler()
scheduler.add_job(run_retries, 'interval', hours=1)
scheduler.start()
app.start()
scheduler.shutdown()
app.start(run_retries())

63
uncia/views.py
View file

@ -1,7 +1,7 @@
import asyncio, json
from izzylib import DotDict, logging
from izzylib.http_server import View
from izzylib.http_server_async import View
from . import __version__
from .config import config
@ -12,24 +12,24 @@ from .processing import ProcessData
### Frontend
class UnciaHome(View):
paths = ['/']
__path__ = '/'
async def get(self, request, response):
with db.session as s:
instances = s.get.instance_list()
return response.template('page/home.haml', {'instances': instances})
response.set_template('page/home.haml', {'instances': instances})
class UnciaAbout(View):
paths = ['/about']
__path__ = '/about'
async def get(self, request, response):
return response.template('page/about.haml')
response.set_template('page/about.haml')
class UnciaRegister(View):
paths = ['/register']
__path__ = '/register'
async def get(self, request, response, error=None, message=None, form={}):
data = {
@ -38,15 +38,15 @@ class UnciaRegister(View):
'message': message
}
return response.template('page/register.haml', data)
response.set_template('page/register.haml', data)
async def post(self, request, response):
return await self.get(request, response, form=request.data.form)
return await self.get(request, response, form=await request.form())
class UnciaLogin(View):
paths = ['/login']
__path__ = '/login'
async def get(self, request, response, error=None, message=None, form={}):
data = {
@ -55,25 +55,25 @@ class UnciaLogin(View):
'message': message
}
return response.template('page/login.haml', data)
response.set_template('page/login.haml', data)
async def post(self, request, response):
return await self.get(request, response, form=request.data.form)
return await self.get(request, response, form=await request.form())
class UnciaLogout(View):
paths = ['/logout']
__path__ = '/logout'
async def get(self, request, response):
return response.redir('/')
response.set_redir('/')
class UnciaUser(View):
paths = ['/user']
__path__ = '/user'
async def get(self, request, response):
return response.template('page/user.haml')
response.set_template('page/user.haml')
async def post(self, request, response):
@ -81,10 +81,10 @@ class UnciaUser(View):
class UnciaAdmin(View):
paths = ['/admin']
__path__ = '/admin'
async def get(self, request, response):
return response.template('page/admin.haml')
response.set_template('page/admin.haml')
async def post(self, request, response):
@ -94,7 +94,7 @@ class UnciaAdmin(View):
### ActivityPub and AP-related endpoints
class UnciaActor(View):
paths = ['/actor', '/inbox']
__path__ = ['/actor', '/inbox']
async def get(self, request, response):
with db.session as s:
@ -125,15 +125,16 @@ class UnciaActor(View):
}
}
return response.json(data)
response.set_json(data, activity=True)
async def post(self, request, response):
if not request.actor:
logging.verbose(f'Failed to fetch actor')
return response.json({'error': 'Failed to fetch actor'})
response.set_json({'error': 'Failed to fetch actor'})
return
processor = ProcessData(request, response, request.data.json)
processor = ProcessData(request, response, await request.json())
if processor.valid_type():
loop = asyncio.get_running_loop()
@ -146,11 +147,12 @@ class UnciaActor(View):
logging.debug(f'Body: {request.text}')
#return response.json({'error': f'Message type unhandled: {processor.type}'}, status=401)
return response.text('UvU', status=202)
response.set_json({'message': 'UvU'})
response.status = 202
class UnciaNodeinfo(View):
paths = ['/nodeinfo/2.0.json', '/nodeinfo/2.0']
__path__ = ['/nodeinfo/2.0.json', '/nodeinfo/2.0']
async def get(self, request, response):
with db.session as s:
@ -190,19 +192,20 @@ class UnciaNodeinfo(View):
}
}
return response.json(data)
response.set_json(data)
class UnciaWebfinger(View):
paths = ['/.well-known/webfinger']
__path__ = '/.well-known/webfinger'
async def get(self, request, response):
resource = request.data.query.get('resource')
resource = request.query.get('resource')
if resource != f'acct:relay@{config.host}':
return response.text('', status=404)
response.status = 404
return
return response.json({
response.set_json({
'subject': f'acct:relay@{config.host}',
'aliases': [
f'https://{config.host}/actor'
@ -218,10 +221,10 @@ class UnciaWebfinger(View):
class UnciaWellknownNodeinfo(View):
paths = ['/.well-known/nodeinfo']
__path__ = '/.well-known/nodeinfo'
async def get(self, request, response):
return response.json({
response.set_json({
'links': [
{
'rel': 'http://nodeinfo.diaspora.software/ns/schema/2.0',