Examples of async reciever tasks

drew/tilt-local-dev
Drew Bednar 2 years ago
parent 6b3bee66c1
commit dde966e650

@ -1,2 +1,4 @@
[MASTER] [MASTER]
ignore-paths=^alembic\\.*$|^alembic/.*$ ignore-paths=^alembic\\.*$|^alembic/.*$
; Needed to prevent E0611: No name 'BaseModel' in module 'pydantic' (no-name-in-module)
extension-pkg-whitelist=pydantic

@ -4,6 +4,6 @@ users_router = APIRouter(
prefix="/users", prefix="/users",
) )
from . import models, tasks # noqa from . import models, tasks, views # noqa
from project.celery_utils import create_celery from project.celery_utils import create_celery

@ -0,0 +1,7 @@
from pydantic import BaseModel
class UserBody(BaseModel):
username: str
email: str

@ -1,11 +1,18 @@
""" """
Many resources on the web recommend using celery.task. This might cause circular imports since you'll have to import the Celery instance. Many resources on the web recommend using celery.task. This might cause circular imports since you'll have to import the Celery instance.
We used shared_task to make our code reusable, which, again, requires current_app in create_celery instead of creating a new Celery instance. We used shared_task to make our code reusable, which, again, requires current_app in create_celery instead of creating a new Celery instance.
Now, we can copy this file anywhere in the app and it will work as expected. Now, we can copy this file anywhere in the app and it will work as expected. In short shared_task lets you define Celery tasks without
having to import the Celery instance, so it can make your task code more reusable.
""" """
import random
import requests
from celery import shared_task from celery import shared_task
from celery.utils.log import get_task_logger
# See https://docs.celeryq.dev/en/stable/userguide/tasks.html#logging
logger = get_task_logger(__name__)
@shared_task @shared_task
def divide(x, y): def divide(x, y):
@ -22,3 +29,31 @@ def add(x, y):
time.sleep(5) time.sleep(5)
return x + y return x + y
@shared_task
def sample_task(email):
"""A sample task simulating calling an external API."""
from project.users.views import api_call
api_call(email)
# Since we set bind to True, this is a bound task, so the first argument to the task
# will always be the current task instance (self). Because of this, we can call self.retry
# to retry the failed task.
@shared_task(bind=True)
def task_process_notification(self):
try:
if not random.choice([0, 1]):
# mimic random error
raise Exception()
# this would block the I/O
requests.post("https://httpbin.org/delay/5", timeout=30)
except Exception as e:
logger.error("exception raised, it would be retry after 5 seconds")
# Remember to raise the exception returned by the self.retry method to make it work.
# By setting the countdown argument to 5, the task will retry after a 5 second delay.
raise self.retry(exc=e, countdown=5)

@ -0,0 +1,122 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Celery example</title>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.2.0/css/bootstrap.min.css"
integrity="sha512-XWTTruHZEYJsxV3W/lSXG1n3Q39YIWOstqvmFsdNEEQfHoZ6vm6E9GK2OrF6DSJSpIbRbi+Nn0WDPID9O7xB2Q=="
crossorigin="anonymous" referrerpolicy="no-referrer"/>
</head>
<body>
<div class="container">
<div class="row">
<div class="col-12 col-md-4">
<form id="your-form">
<div class="mb-3">
<label for="email" class="form-label">Email address</label>
<input type="email" class="form-control" id="email" name="email">
</div>
<div class="mb-3">
<label for="username" class="form-label">Username</label>
<input type="text" class="form-control" id="username" name="username">
</div>
<div class="mb-3" id="messages"></div>
<button type="submit" class="btn btn-primary">Submit</button>
</form>
</div>
</div>
</div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.2.0/js/bootstrap.min.js"
integrity="sha512-8Y8eGK92dzouwpROIppwr+0kPauu0qqtnzZZNEF8Pat5tuRNJxJXCkbQfJ0HlUG3y1HB3z18CSKmUo7i2zcPpg=="
crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script>
function updateProgress(yourForm, task_id, btnHtml) {
fetch(`/users/task_status/?task_id=${task_id}`, {
method: 'GET',
})
.then(response => response.json())
.then((res) => {
const taskStatus = res.state;
if (['SUCCESS', 'FAILURE'].includes(taskStatus)) {
const msg = yourForm.querySelector('#messages');
const submitBtn = yourForm.querySelector('button[type="submit"]');
if (taskStatus === 'SUCCESS') {
msg.innerHTML = 'job succeeded';
} else if (taskStatus === 'FAILURE') {
// display error message on the form
msg.innerHTML = res.error;
}
submitBtn.disabled = false;
submitBtn.innerHTML = btnHtml;
} else {
// the task is still running
setTimeout(function() {
updateProgress(yourForm, task_id, btnHtml);
}, 1000);
}
})
.catch((error) => {
console.error('Error:', error)
});
}
function serialize(data) {
let obj = {};
for (let [key, value] of data) {
if (obj[key] !== undefined) {
if (!Array.isArray(obj[key])) {
obj[key] = [obj[key]];
}
obj[key].push(value);
} else {
obj[key] = value;
}
}
return obj;
}
document.addEventListener("DOMContentLoaded", function() {
const yourForm = document.getElementById("your-form");
yourForm.addEventListener("submit", function(event) {
event.preventDefault();
const submitBtn = yourForm.querySelector('button[type="submit"]');
const btnHtml = submitBtn.innerHTML;
const spinnerHtml = 'Processing...';
submitBtn.disabled = true;
submitBtn.innerHTML = spinnerHtml;
const msg = yourForm.querySelector('#messages');
msg.innerHTML = '';
// Get all field data from the form
let data = new FormData(yourForm);
// Convert to an object
let formData = serialize(data);
fetch('/users/form/', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(formData),
})
.then(response => response.json())
.then((res) => {
// after we get Celery task id, we start polling
const task_id = res.task_id;
updateProgress(yourForm, task_id, btnHtml);
console.log(res);
})
.catch((error) => {
console.error('Error:', error)
});
});
});
</script>
</body>
</html>

@ -0,0 +1,71 @@
import logging
import random
import requests
from celery.result import AsyncResult
from fastapi import Request
from fastapi.responses import JSONResponse
from fastapi.templating import Jinja2Templates
from . import users_router
from .schema import UserBody
from .tasks import sample_task, task_process_notification
logger = logging.getLogger(__name__)
templates = Jinja2Templates(directory="project/users/templates")
def api_call(email: str): # pylint: disable=unused-argument
# used for testing a failed api call
if random.choice([0, 1]):
raise Exception("random processing error")
# used for simulating a call to a third-party api
requests.post("https://httpbin.org/delay/5", timeout=30)
@users_router.get("/form/")
def form_example_get(request: Request):
return templates.TemplateResponse("form.html", {"request": request})
@users_router.post("/form/")
def form_example_post(user_body: UserBody):
task = sample_task.delay(user_body.email)
return JSONResponse({"task_id": task.task_id})
@users_router.get("/task_status/")
def task_status(task_id: str):
task = AsyncResult(task_id)
state = task.state
if state == "FAILURE":
error = str(task.result)
response = {
"state": state,
"error": error,
}
else:
response = {
"state": state,
}
return JSONResponse(response)
@users_router.post("/webhook_test/")
def webhook_test():
if not random.choice([0, 1]):
# mimic an error
raise Exception()
# blocking process
requests.post("https://httpbin.org/delay/5", timeout=30)
return "pong"
@users_router.post("/webhook_test_async/")
def webhook_test_async():
task = task_process_notification.delay()
print(task.id)
return "pong"

@ -2,9 +2,10 @@ alembic==1.8.1
celery==5.2.7 celery==5.2.7
fastapi==0.79.0 fastapi==0.79.0
flower==1.2.0 flower==1.2.0
Jinja2==3.1.2
psycopg2-binary==2.9.3 psycopg2-binary==2.9.3
redis==4.3.4 redis==4.3.4
shellcheck-py==0.8.0.4 requests==2.28.1
SQLAlchemy==1.4.40 SQLAlchemy==1.4.40
uvicorn[standard]==0.18.2 uvicorn[standard]==0.18.2
watchfiles==0.17.0 watchfiles==0.17.0

@ -20,6 +20,10 @@ celery==5.2.7
# via # via
# -r requirements.in # -r requirements.in
# flower # flower
certifi==2022.9.24
# via requests
charset-normalizer==2.1.1
# via requests
click==8.1.3 click==8.1.3
# via # via
# celery # celery
@ -48,17 +52,23 @@ httptools==0.5.0
humanize==4.4.0 humanize==4.4.0
# via flower # via flower
idna==3.4 idna==3.4
# via anyio # via
# anyio
# requests
importlib-metadata==4.12.0 importlib-metadata==4.12.0
# via alembic # via alembic
importlib-resources==5.9.0 importlib-resources==5.9.0
# via alembic # via alembic
jinja2==3.1.2
# via -r requirements.in
kombu==5.2.4 kombu==5.2.4
# via celery # via celery
mako==1.2.2 mako==1.2.2
# via alembic # via alembic
markupsafe==2.1.1 markupsafe==2.1.1
# via mako # via
# jinja2
# mako
packaging==21.3 packaging==21.3
# via redis # via redis
prometheus-client==0.14.1 prometheus-client==0.14.1
@ -81,6 +91,8 @@ pyyaml==6.0
# via uvicorn # via uvicorn
redis==4.3.4 redis==4.3.4
# via -r requirements.in # via -r requirements.in
requests==2.28.1
# via -r requirements.in
six==1.16.0 six==1.16.0
# via click-repl # via click-repl
sniffio==1.3.0 sniffio==1.3.0
@ -97,6 +109,8 @@ typing-extensions==4.3.0
# via # via
# pydantic # pydantic
# starlette # starlette
urllib3==1.26.12
# via requests
uvicorn[standard]==0.18.2 uvicorn[standard]==0.18.2
# via -r requirements.in # via -r requirements.in
uvloop==0.17.0 uvloop==0.17.0

Loading…
Cancel
Save