backfill-over-federation from sqlite servers is completely broken

[richvdh]

since 0.27.0-rc1, apparently (introduced by #2990)

[richvdh]

2018-06-06 13:59:18,444 - 140620972586752 synapse.http.server - 102 - ERROR - [[GET-0]] Failed handle request via <function _async_render at 0x7fe4d793baa0>: <SynapseRequest at 0x7fe4d79ab368 method='GET' uri='/_matrix/federation/v1/backfill/%21pfckadQIhQBjSxSreH%3Afred.sw1v.org/?limit=100&v=%24152828448611PwllX%3Afred.sw1v.org' clientproto='HTTP/1.1' site=8448>: Traceback (most recent call last):
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1442, in gotResult
    _inlineCallbacks(r, g, deferred)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
--- <exception caught here> ---
  File "synapse/http/server.py", line 79, in wrapped_request_handler
    yield h(self, request)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/http/server.py", line 305, in _async_render
    callback_return = yield callback(request, **kwargs)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/federation/transport/server.py", line 203, in new_func
    origin, content, request.args, *args, **kwargs
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/federation/federation_server.py", line 78, in on_backfill_request
    origin, room_id, versions, limit
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/handlers/federation.py", line 1465, in on_backfill_request
    events = yield self._filter_events_for_server(origin, room_id, events)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/util/metrics.py", line 55, in measured_func
    r = yield func(self, *args, **kwargs)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/handlers/federation.py", line 465, in _filter_events_for_server
    (EventTypes.Member, None),
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/storage/state.py", line 393, in get_state_ids_for_events
    group_to_state = yield self._get_state_for_groups(groups, types)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/storage/state.py", line 565, in _get_state_for_groups
    missing_groups, types
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/storage/state.py", line 196, in _get_state_groups_from_groups
    self._get_state_groups_from_groups_txn, chunk, types,
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/storage/_base.py", line 325, in runInteraction
    result = yield self.runWithConnection(inner_func, *args, **kwargs)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "synapse/storage/_base.py", line 370, in runWithConnection
    inner_func, *args, **kwargs
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/threadpool.py", line 250, in inContext
    result = inContext.theWork()
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/threadpool.py", line 266, in <lambda>
    inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/context.py", line 122, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/context.py", line 85, in callWithContext
    return func(*args,**kw)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/enterprise/adbapi.py", line 307, in _runWithConnection
    compat.reraise(excValue, excTraceback)
  File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/enterprise/adbapi.py", line 298, in _runWithConnection
    result = func(conn, *args, **kw)
  File "synapse/storage/_base.py", line 366, in inner_func
    return func(conn, *args, **kwargs)
  File "synapse/storage/_base.py", line 321, in inner_func
    func, *args, **kwargs
  File "synapse/storage/_base.py", line 246, in _new_transaction
    r = func(txn, *args, **kwargs)
  File "synapse/storage/state.py", line 304, in _get_state_groups_from_groups_txn
    args
  File "synapse/storage/_base.py", line 87, in execute
    self._do_execute(self.txn.execute, sql, *args)
  File "synapse/storage/_base.py", line 117, in _do_execute
    sql, *args
sqlite3.ProgrammingError: Incorrect number of bindings supplied. The current statement uses 4, and there are 16 supplied.

[ara4n]

shit, sorry :(

…

-- Matthew Hodgson Matrix.org

On 6 Jun 2018, at 16:14, Richard van der Hoff ***@***.***> wrote: 2018-06-06 13:59:18,444 - 140620972586752 synapse.http.server - 102 - ERROR - [[GET-0]] Failed handle request via <function _async_render at 0x7fe4d793baa0>: <SynapseRequest at 0x7fe4d79ab368 method='GET' uri='/_matrix/federation/v1/backfill/%21pfckadQIhQBjSxSreH%3Afred.sw1v.org/?limit=100&v=%24152828448611PwllX%3Afred.sw1v.org' clientproto='HTTP/1.1' site=8448>: Traceback (most recent call last): File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 653, in _runCallbacks current.result = callback(current.result, *args, **kw) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1442, in gotResult _inlineCallbacks(r, g, deferred) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) --- <exception caught here> --- File "synapse/http/server.py", line 79, in wrapped_request_handler yield h(self, request) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/http/server.py", line 305, in _async_render callback_return = yield callback(request, **kwargs) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/federation/transport/server.py", line 203, in new_func origin, content, request.args, *args, **kwargs File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/federation/federation_server.py", line 78, in on_backfill_request origin, room_id, versions, limit File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/handlers/federation.py", line 1465, in on_backfill_request events = yield self._filter_events_for_server(origin, room_id, events) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/util/metrics.py", line 55, in measured_func r = yield func(self, *args, **kwargs) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/handlers/federation.py", line 465, in _filter_events_for_server (EventTypes.Member, None), File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/storage/state.py", line 393, in get_state_ids_for_events group_to_state = yield self._get_state_for_groups(groups, types) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/storage/state.py", line 565, in _get_state_for_groups missing_groups, types File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/storage/state.py", line 196, in _get_state_groups_from_groups self._get_state_groups_from_groups_txn, chunk, types, File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/storage/_base.py", line 325, in runInteraction result = yield self.runWithConnection(inner_func, *args, **kwargs) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1384, in _inlineCallbacks result = result.throwExceptionIntoGenerator(g) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/failure.py", line 408, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) File "synapse/storage/_base.py", line 370, in runWithConnection inner_func, *args, **kwargs File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/threadpool.py", line 250, in inContext result = inContext.theWork() File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/threadpool.py", line 266, in <lambda> inContext.theWork = lambda: context.call(ctx, func, *args, **kw) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/context.py", line 122, in callWithContext return self.currentContext().callWithContext(ctx, func, *args, **kw) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/python/context.py", line 85, in callWithContext return func(*args,**kw) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/enterprise/adbapi.py", line 307, in _runWithConnection compat.reraise(excValue, excTraceback) File "/home/rav/work/synapse/env/local/lib/python2.7/site-packages/twisted/enterprise/adbapi.py", line 298, in _runWithConnection result = func(conn, *args, **kw) File "synapse/storage/_base.py", line 366, in inner_func return func(conn, *args, **kwargs) File "synapse/storage/_base.py", line 321, in inner_func func, *args, **kwargs File "synapse/storage/_base.py", line 246, in _new_transaction r = func(txn, *args, **kwargs) File "synapse/storage/state.py", line 304, in _get_state_groups_from_groups_txn args File "synapse/storage/_base.py", line 87, in execute self._do_execute(self.txn.execute, sql, *args) File "synapse/storage/_base.py", line 117, in _do_execute sql, *args sqlite3.ProgrammingError: Incorrect number of bindings supplied. The current statement uses 4, and there are 16 supplied. — You are receiving this because you are subscribed to this thread. Reply to this email directly, view it on GitHub, or mute the thread.

[richvdh]

Nm. I think the main thing this ponts to is the woeful lack of test coverage.

[richvdh]

fixed by #3355