create sql statements for querysets the code in here encapsulates all of the sql construction so that querysets themselves do not have to and could be backed by things other than sql databases the abstraction barrier only works one way this module has to know all about the internals of models in order to get the information it needs quotation marks whitespace characters semicolons or inline sql comments are forbidden in column aliases inspired from https www postgresql orgdocscurrentsqlsyntaxlexical htmlsqlsyntaxidentifiers a single raw sql query def initself sql using params self params params self sql sql self using using self cursor none mirror some properties of a normal query so that the compiler can be used to process results self lowmark self highmark 0 none used for offsetlimit self extraselect self annotationselect def chainself using return self cloneusing def cloneself using return rawqueryself sql using paramsself params def getcolumnsself if self cursor is none self executequery converter connectionsself using introspection identifierconverter return convertercolumnmeta0 for columnmeta in self cursor description def iterself always execute a new query for a new iterator this could be optimized with a cache at the expense of ram self executequery if not connectionsself using features canusechunkedreads if the database can t use chunked reads we need to make sure we evaluate the entire query up front result listself cursor else result self cursor return iterresult def reprself return s s self class name self property def paramstypeself if self params is none return none return dict if isinstanceself params mapping else tuple def strself if self paramstype is none return self sql return self sql self paramstypeself params def executequeryself connection connectionsself using adapt parameters to the database as much as possible considering that the target type isn t known see 17755 paramstype self paramstype adapter connection ops adaptunknownvalue if paramstype is tuple params tupleadapterval for val in self params elif paramstype is dict params key adapterval for key val in self params items elif paramstype is none params none else raise runtimeerrorunexpected params type s paramstype self cursor connection cursor self cursor executeself sql params explaininfo namedtupleexplaininfo format options class querybaseexpression sqlrelated attributes select and related select clauses are expressions to use in the select clause of the query the select is used for cases where we want to set up the select clause to contain other than default fields values subqueries note that annotations go to annotations dictionary the groupby attribute can have one of the following forms none no group by at all in the query a tuple of expressions group by at least those expressions string refs are also allowed for now true group by all select fields of the model see compiler getgroupby for details arbitrary limit for selectrelated to prevents infinite recursion holds the selects defined by a call to values or valueslist excluding annotationselect and extraselect sql annotationrelated attributes set combination attributes these are for extensions the contents are more or less appended verbatim to the appropriate clause a tuple that is a set of model field names and either true if these are the fields to defer or false if these are the only fields to load aliasmap is the most important data structure regarding joins it s used for recording which joins exist in the query and what types they are the key is the alias of the joined table possibly the table name and the value is a joinlike object see sql datastructures join for more information whether to provide alias to columns during reference resolving sometimes the query contains references to aliases in outer queries as a result of splitexclude correct alias quoting needs to know these aliases too map external tables to whether they are aliased maps alias annotation expression these are for extensions the contents are more or less appended verbatim to the appropriate clause return the query as a string of sql with the parameter values substituted in use sqlwithparams to see the unsubstituted string parameter values won t necessarily be quoted correctly since that is done by the database interface at execution time return the query as an sql string and the parameters that will be substituted into the query limit the amount of work when a query is deepcopied result self clone memoidself result return result def getcompilerself usingnone connectionnone elideemptytrue if using is none and connection is none raise valueerrorneed either using or connection if using connection connectionsusing return connection ops compilerself compiler self connection using elideempty def getmetaself if self model return self model meta def cloneself obj empty obj class self class copy references to everything obj dict self dict copy clone attributes that can t use shallow copy obj aliasrefcount self aliasrefcount copy obj aliasmap self aliasmap copy obj externalaliases self externalaliases copy obj tablemap self tablemap copy obj where self where clone obj annotations self annotations copy if self annotationselectmask is not none obj annotationselectmask self annotationselectmask copy if self combinedqueries obj combinedqueries tuple query clone for query in self combinedqueries annotationselectcache cannot be copied as doing so breaks the necessary state in which both annotations and annotationselectcache point to the same underlying objects it will get repopulated in the cloned queryset the next time it s used obj annotationselectcache none obj extra self extra copy if self extraselectmask is not none obj extraselectmask self extraselectmask copy if self extraselectcache is not none obj extraselectcache self extraselectcache copy if self selectrelated is not false use deepcopy because selectrelated stores fields in nested dicts obj selectrelated copy deepcopyobj selectrelated if subqaliases in self dict obj subqaliases self subqaliases copy obj usedaliases self usedaliases copy obj filteredrelations self filteredrelations copy clear the cachedproperty if it exists obj dict popbasetable none return obj def chainself klassnone obj self clone if klass and obj class klass obj class klass if not obj filterissticky obj usedaliases set obj filterissticky false if hasattrobj setupquery obj setupquery return obj def relabeledcloneself changemap clone self clone clone changealiaseschangemap return clone def getcolself target field alias if not self aliascols alias none return target getcolalias field def getaggregationself using aggregateexprs if not aggregateexprs return store annotation mask prior to temporarily adding aggregations for resolving purpose to facilitate their subsequent removal refssubquery false refswindow false replacements annotationselectmask self annotationselectmask for alias aggregateexpr in aggregateexprs items self checkaliasalias aggregate aggregateexpr resolveexpression self allowjoinstrue reusenone summarizetrue if not aggregate containsaggregate raise typeerrors is not an aggregate expression alias temporarily add aggregate to annotations to allow remaining members of aggregates to resolve against each others self appendannotationmaskalias aggregaterefs aggregate getrefs refssubquery any getattrself annotationsref containssubquery false for ref in aggregaterefs refswindow any getattrself annotationsref containsoverclause true for ref in aggregaterefs aggregate aggregate replaceexpressionsreplacements self annotationsalias aggregate replacementsrefalias aggregate aggregate stash resolved aggregates now that they have been allowed to resolve against each other aggregates alias self annotations popalias for alias in aggregateexprs self setannotationmaskannotationselectmask existing usage of aggregation can be determined by the presence of selected aggregates but also by filters against aliased aggregates having qualify self where splithavingqualify hasexistingaggregation any getattrannotation containsaggregate true for annotation in self annotations values or having decide if we need to use a subquery existing aggregations would cause incorrect results as getaggregation must produce just one result and thus must not use group by if the query has limit or distinct or uses set operations then those operations must be done in a subquery so that the query aggregates on the limit andor distinct results instead of applying the distinct and limit after the aggregation if isinstanceself groupby tuple or self issliced or hasexistingaggregation or refssubquery or refswindow or qualify or self distinct or self combinator from django db models sql subqueries import aggregatequery innerquery self clone innerquery subquery true outerquery aggregatequeryself model innerquery innerquery selectforupdate false innerquery selectrelated false innerquery setannotationmaskself annotationselect queries with distinctfields need ordering and when a limit is applied we must take the slice from the ordered query otherwise no need for ordering innerquery clearorderingforcefalse if not innerquery distinct if the inner query uses default select and it has some aggregate annotations then we must make sure the inner query is grouped by the main model s primary key however clearing the select clause can alter results if distinct is used if innerquery defaultcols and hasexistingaggregation innerquery groupby self model meta pk getcolinnerquery getinitialalias innerquery defaultcols false if not qualify mask existing annotations that are not referenced by aggregates to be pushed to the outer query unless filtering against window functions is involved as it requires complex realising annotationmask set if isinstanceself groupby tuple for expr in self groupby annotationmask expr getrefs for aggregate in aggregates values annotationmask aggregate getrefs avoid eliding expressions that might have an incidence on the implicit grouping logic for annotationalias annotation in self annotationselect items if annotation getgroupbycols annotationmask addannotationalias innerquery setannotationmaskannotationmask add aggregates to the outer aggregatequery this requires making sure all columns referenced by the aggregates are selected in the inner query it is achieved by retrieving all column references by the aggregates explicitly selecting them in the inner query and making sure the aggregates are repointed to them colrefs for alias aggregate in aggregates items replacements for col in self gencolsaggregate resolverefsfalse if not colref colrefs getcol index lencolrefs 1 colalias fcolindex colref refcolalias col colrefscol colref innerquery annotationscolalias col innerquery appendannotationmaskcolalias replacementscol colref outerquery annotationsalias aggregate replaceexpressions replacements if innerquery select and not innerquery defaultcols and not innerquery annotationselectmask in case of model objects0 3 count there would be no field selected in the inner query yet we must use a subquery so make sure at least one field is selected innerquery select self model meta pk getcolinnerquery getinitialalias else outerquery self self select self defaultcols false self extra if self annotations inline reference to existing annotations and mask them as they are unnecessary given only the summarized aggregations are requested replacements refalias annotation annotation for alias annotation in self annotations items self annotations alias aggregate replaceexpressionsreplacements for alias aggregate in aggregates items else self annotations aggregates self setannotationmaskaggregates emptysetresult expression emptyresultsetvalue for expression in outerquery annotationselect values elideempty not anyresult is notimplemented for result in emptysetresult outerquery clearorderingforcetrue outerquery clearlimits outerquery selectforupdate false outerquery selectrelated false compiler outerquery getcompilerusing elideemptyelideempty result compiler executesqlsingle if result is none result emptysetresult else converters compiler getconvertersouterquery annotationselect values result nextcompiler applyconvertersresult converters return dictzipouterquery annotationselect result def getcountself using obj self clone return obj getaggregationusing count countcount def hasfiltersself return self where def existsself limittrue q self clone if not q distinct and q issliced if q groupby is true q addfields f attname for f in self model meta concretefields false disable group by aliases to avoid orphaning references to the select clause which is about to be cleared q setgroupbyallowaliasesfalse q clearselectclause if q combinedqueries and q combinator union q combinedqueries tuple combinedquery existslimitfalse for combinedquery in q combinedqueries q clearorderingforcetrue if limit q setlimitshigh1 q addannotationvalue1 a return q def hasresultsself using q self existsusing compiler q getcompilerusingusing return compiler hasresults def explainself using formatnone options q self clone for optionname in options if not explainoptionspattern fullmatchoptionname or in optionname raise valueerrorfinvalid option name optionname r q explaininfo explaininfoformat options compiler q getcompilerusingusing return n joincompiler explainquery def combineself rhs connector if self model rhs model raise typeerrorcannot combine queries on two different base models if self issliced raise typeerrorcannot combine queries once a slice has been taken if self distinct rhs distinct raise typeerrorcannot combine a unique query with a nonunique query if self distinctfields rhs distinctfields raise typeerrorcannot combine queries with different distinct fields if lhs and rhs shares the same alias prefix it is possible to have conflicting alias changes like t4 t5 t5 t6 which might end up as t4 t6 while combining two querysets to prevent this change an alias prefix of the rhs and update current aliases accordingly except if the alias is the base table since it must be present in the query on both sides initialalias self getinitialalias rhs bumpprefixself excludeinitialalias work out how to relabel the rhs aliases if necessary changemap conjunction connector and determine which existing joins can be reused when combining the query with and we must recreate all joins for m2m filters when combining with or we can reuse joins the reason is that in and case a single row can t fulfill a condition like revrelcol1 revrelcol2 but there might be two different related rows matching this condition in or case a single true is enough so single row is enough too note that we will be creating duplicate joins for nonm2m joins in the and case the results will be correct but this creates too many joins this is something that could be fixed later on reuse set if conjunction else setself aliasmap joinpromoter joinpromoterconnector 2 false joinpromoter addvotes j for j in self aliasmap if self aliasmapj jointype inner rhsvotes set now add the joins from rhs query into the new query skipping base table rhstables listrhs aliasmap1 for alias in rhstables join rhs aliasmapalias if the left side of the join was already relabeled use the updated alias join join relabeledclonechangemap newalias self joinjoin reusereuse if join jointype inner rhsvotes addnewalias we can t reuse the same join again in the query if we have two distinct joins for the same connection in rhs query then the combined query must have two joins too reuse discardnewalias if alias newalias changemapalias newalias if not rhs aliasrefcountalias the alias was unused in the rhs query unref it so that it will be unused in the new query too we have to add and unref the alias so that join promotion has information of the join type for the unused alias self unrefaliasnewalias joinpromoter addvotesrhsvotes joinpromoter updatejointypesself combine subqueries aliases to ensure aliases relabelling properly handle subqueries when combining where and select clauses self subqaliases rhs subqaliases now relabel a copy of the rhs whereclause and add it to the current one w rhs where clone w relabelaliaseschangemap self where addw connector selection columns and extra extensions are those provided by rhs if rhs select self setselectcol relabeledclonechangemap for col in rhs select else self select if connector or it would be nice to be able to handle this but the queries don t really make sense or return consistent value sets not worth the extra complexity when you can write a real query instead if self extra and rhs extra raise valueerror when merging querysets using or you cannot have extraselect on both sides self extra updaterhs extra extraselectmask set if self extraselectmask is not none extraselectmask updateself extraselectmask if rhs extraselectmask is not none extraselectmask updaterhs extraselectmask if extraselectmask self setextramaskextraselectmask self extratables rhs extratables ordering uses the rhs ordering unless it has none in which case the current ordering is used self orderby rhs orderby or self orderby self extraorderby rhs extraorderby or self extraorderby def getdeferselectmaskself opts mask selectmasknone if selectmask is none selectmask selectmaskopts pk all concrete fields that are not part of the defer mask must be loaded if a relational field is encountered it gets added to the mask for it be considered if selectrelated and the cycle continues by recursively calling this function for field in opts concretefields fieldmask mask popfield name none fieldattmask mask popfield attname none if fieldmask is none and fieldattmask is none selectmask setdefaultfield elif fieldmask if not field isrelation raise fielderrornextiterfieldmask fieldselectmask selectmask setdefaultfield relatedmodel field remotefield model meta concretemodel self getdeferselectmask relatedmodel meta fieldmask fieldselectmask remaining defer entries must be references to reverse relationships the following code is expected to raise fielderror if it encounters a malformed defer entry for fieldname fieldmask in mask items if filteredrelation self filteredrelations getfieldname relation opts getfieldfilteredrelation relationname fieldselectmask selectmask setdefaultfieldname relation field relation field else reverserel opts getfieldfieldname while virtual fields such as manytomany and generic foreign keys cannot be effectively deferred we ve historically allowed them to be passed to queryset defer ignore such field references until a layer of validation at mask alteration time will be implemented eventually if not hasattrreverserel field continue field reverserel field fieldselectmask selectmask setdefaultfield relatedmodel field model meta concretemodel self getdeferselectmask relatedmodel meta fieldmask fieldselectmask return selectmask def getonlyselectmaskself opts mask selectmasknone if selectmask is none selectmask selectmaskopts pk only include fields mentioned in the mask for fieldname fieldmask in mask items field opts getfieldfieldname retrieve the actual field associated with reverse relationships as that s what is expected in the select mask if field in opts relatedobjects fieldkey field field else fieldkey field fieldselectmask selectmask setdefaultfieldkey if fieldmask if not field isrelation raise fielderrornextiterfieldmask relatedmodel field remotefield model meta concretemodel self getonlyselectmask relatedmodel meta fieldmask fieldselectmask return selectmask def getselectmaskself fieldnames defer self deferredloading if not fieldnames return mask for fieldname in fieldnames partmask mask for part in fieldname splitlookupsep partmask partmask setdefaultpart opts self getmeta if defer return self getdeferselectmaskopts mask return self getonlyselectmaskopts mask def tablealiasself tablename createfalse filteredrelationnone aliaslist self tablemap gettablename if not create and aliaslist alias aliaslist0 self aliasrefcountalias 1 return alias false create a new alias for this table if aliaslist alias sd self aliasprefix lenself aliasmap 1 aliaslist appendalias else the first occurrence of a table uses the table name directly alias filteredrelation alias if filteredrelation is not none else tablename self tablemaptablename alias self aliasrefcountalias 1 return alias true def refaliasself alias decreases the reference count for this alias self aliasrefcountalias amount def promotejoinsself aliases aliases listaliases while aliases alias aliases pop0 if self aliasmapalias jointype is none this is the base table first from entry this table isn t really joined at all in the query so we should not alter its join type continue only the first alias skipped above should have none jointype assert self aliasmapalias jointype is not none parentalias self aliasmapalias parentalias parentlouter parentalias and self aliasmapparentalias jointype louter alreadylouter self aliasmapalias jointype louter if self aliasmapalias nullable or parentlouter and not alreadylouter self aliasmapalias self aliasmapalias promote join type of alias changed so reexamine all aliases that refer to this one aliases extend join for join in self aliasmap if self aliasmapjoin parentalias alias and join not in aliases def demotejoinsself aliases aliases listaliases while aliases alias aliases pop0 if self aliasmapalias jointype louter self aliasmapalias self aliasmapalias demote parentalias self aliasmapalias parentalias if self aliasmapparentalias jointype inner aliases appendparentalias def resetrefcountsself tocounts for alias currefcount in self aliasrefcount copy items unrefamount currefcount tocounts getalias 0 self unrefaliasalias unrefamount def changealiasesself changemap if keys and values of changemap were to intersect an alias might be updated twice e g t4 t5 t5 t6 so also t4 t6 depending on their order in changemap assert setchangemap isdisjointchangemap values 1 update references in select normal columns plus aliases group by and where self where relabelaliaseschangemap if isinstanceself groupby tuple self groupby tuple col relabeledclonechangemap for col in self groupby self select tuplecol relabeledclonechangemap for col in self select self annotations self annotations and key col relabeledclonechangemap for key col in self annotations items 2 rename the alias in the internal tablealias datastructures for oldalias newalias in changemap items if oldalias not in self aliasmap continue aliasdata self aliasmapoldalias relabeledclonechangemap self aliasmapnewalias aliasdata self aliasrefcountnewalias self aliasrefcountoldalias del self aliasrefcountoldalias del self aliasmapoldalias tablealiases self tablemapaliasdata tablename for pos alias in enumeratetablealiases if alias oldalias tablealiasespos newalias break self externalaliases table is aliased or it s being changed and thus is aliased changemap getalias alias aliased or alias in changemap for alias aliased in self externalaliases items def bumpprefixself otherquery excludenone def prefixgen alphabet asciiuppercase prefix chrordself aliasprefix 1 yield prefix for n in count1 seq alphabetalphabet indexprefix if prefix else alphabet for s in productseq repeatn yield joins prefix none if self aliasprefix otherquery aliasprefix no clashes between self and outer query should be possible return explicitly avoid infinite loop the constant divider is based on how much depth recursive subquery references add to the stack this value might need to be adjusted when adding or removing function calls from the code path in charge of performing these operations localrecursionlimit sys getrecursionlimit 16 for pos prefix in enumerateprefixgen if prefix not in self subqaliases self aliasprefix prefix break if pos localrecursionlimit raise recursionerror maximum recursion depth exceeded too many subqueries self subqaliases self subqaliases unionself aliasprefix otherquery subqaliases otherquery subqaliases unionself subqaliases if exclude is none exclude self changealiases alias sd self aliasprefix pos for pos alias in enumerateself aliasmap if alias not in exclude def getinitialaliasself if self aliasmap alias self basetable self refaliasalias elif self model alias self joinself basetableclassself getmeta dbtable none else alias none return alias def countactivetablesself return len1 for count in self aliasrefcount values if count def joinself join reusenone reusealiases a for a j in self aliasmap items if reuse is none or a in reuse and j join if reusealiases if join tablealias in reusealiases reusealias join tablealias else reuse the most recent alias of the joined table a manytomany relation may be joined multiple times reusealias reusealiases1 self refaliasreusealias return reusealias no reuse is possible so we need a new alias alias self tablealias join tablename createtrue filteredrelationjoin filteredrelation if join jointype if self aliasmapjoin parentalias jointype louter or join nullable jointype louter else jointype inner join jointype jointype join tablealias alias self aliasmapalias join if filteredrelation join filteredrelation resolvereuse reuse if resolvereuse is not none resolvereuse setreuse alias joinslen lenself aliasmap join filteredrelation filteredrelation resolveexpression self reuseresolvereuse some joins were during expression resolving they must be present before the one we just added if joinslen lenself aliasmap self aliasmapalias self aliasmap popalias return alias def joinparentmodelself opts model alias seen if model in seen return seenmodel chain opts getbasechainmodel if not chain return alias curropts opts for intmodel in chain if intmodel in seen curropts intmodel meta alias seenintmodel continue proxy model have elements in base chain with no parents assign the new options object and skip to the next base in that case if not curropts parentsintmodel curropts intmodel meta continue linkfield curropts getancestorlinkintmodel joininfo self setupjoinslinkfield name curropts alias curropts intmodel meta alias seenintmodel joininfo joins1 return alias or seennone def checkaliasself alias if forbiddenaliaspattern searchalias raise valueerror column aliases cannot contain whitespace characters quotation marks semicolons or sql comments def addannotationself annotation alias selecttrue subqueries need to use a different set of aliases than the outer query resolve combined queries outer query s aliases are considered external if wrapper is referenced by an alias for an explicit group by through values a reference to this expression and not the self must be returned to ensure external column references are not grouped against as well some backends e g oracle raise an error when a subquery contains unnecessary order by clause the items of the iterable may be expressions and therefore need to be resolved independently solve the lookup type from the lookup e g foobaridicontains check whether the object passed while querying is of the correct type if not raise a valueerror specifying the wrong object check the type of object passed to query relations if field isrelation check that the field and the queryset use the same model in a query like filter objects all for example the opts would be s from the field and value model would be objects all queryset s model also the field is the related field on the lhs side if isinstancevalue query and not value hasselectfields and not checkrellookupcompatibilityvalue model opts field raise valueerror cannot use queryset for s use a queryset for s value model meta objectname opts objectname elif hasattrvalue meta self checkqueryobjecttypevalue opts field elif hasattrvalue iter for v in value self checkqueryobjecttypev opts field def checkfilterableself expression try to extract transforms and lookup from given lhs the lhs value is something that works like sqlexpression the rhs value is what the lookup is going to compare against the lookups is a list of names to extract using getlookup and gettransform exact is the default lookup if one isn t given first try getlookup so that the lookup takes precedence if the lhs supports both transform and lookup for the name a lookup wasn t found try to interpret the name as a transform and do an exact lookup against it interpret exactnone as the sql is null otherwise reject all uses of none as a query value unless the lookup supports it for oracle is equivalent to null the check must be done at this stage because join promotion can t be done in the compiler using defaultdbalias isn t nice but it s the best that can be done here a similar thing is done in isnullable too helper method for buildlookup try to fetch and initialize a transform for name parameter from lhs build a wherenode for a single filter clause but don t add it to this query query addq will then add this filter to the where node the branchnegated tells us if the current branch contains any negations this will be used to determine if subqueries are needed the currentnegated is used to determine if the current filter is negated or not and this will be used to determine if is null filtering is needed the difference between currentnegated and branchnegated is that branchnegated is set on first negation but currentnegated is flipped for each negation note that addfilter will not do any negating itself that is done upper in the code by addq the canreuse is a set of reusable joins for multijoins the method will create a filter clause that can be added to the current query however if the filter isn t added to the query then the caller is responsible for unreffing the joins used prevent iterator from being consumed by checkrelatedobjects splitexclude needs to know which joins were generated for the lookup parts update usedjoins before trimming since they are reused to determine which joins could be later promoted to inner the condition added here will be sql like this not col is not null where the first not is added in upper layers of code the reason for addition is that if col is null then col someval will result in sql unknown which isn t the same as in python the python none handling is wanted and it can be gotten by col is null or col someval not col is not null and col someval if someval is a nullable column someval is not null is added a preprocessor for the internal addq responsible for doing final join promotion for join promotion this case is doing an and for the added qobject and existing conditions so any existing inner join forces the join type to remain inner existing outer joins can however be demoted consider case where rela is louter and relacol1 is added if rela doesn t produce any rows then the whole condition must fail so demotion is ok add a qobject to the current filter connector qobject connector currentnegated qobject negated branchnegated branchnegated or qobject negated targetclause wherenodeconnectorconnector negatedqobject negated joinpromoter joinpromoter qobject connector lenqobject children currentnegated for child in qobject children childclause neededinner self buildfilter child canreuseusedaliases branchnegatedbranchnegated currentnegatedcurrentnegated allowjoinsallowjoins splitsubqsplitsubq checkfilterablecheckfilterable summarizesummarize updatejointypesupdatejointypes joinpromoter addvotesneededinner if childclause targetclause addchildclause connector if updatejointypes neededinner joinpromoter updatejointypesself else neededinner return targetclause neededinner def addfilteredrelationself filteredrelation alias filteredrelation alias alias relationlookupparts relationfieldparts self solvelookuptype filteredrelation relationname if relationlookupparts raise valueerror filteredrelation s relationname cannot contain lookups got r filteredrelation relationname for lookup in getchildrenfromqfilteredrelation condition lookupparts lookupfieldparts self solvelookuptypelookup shift 2 if not lookupparts else 1 lookupfieldpath lookupfieldparts shift for idx lookupfieldpart in enumeratelookupfieldpath if lenrelationfieldparts idx if relationfieldpartsidx lookupfieldpart raise valueerror filteredrelation s condition doesn t support relations outside the r got r filteredrelation relationname lookup else raise valueerror filteredrelation s condition doesn t support nested relations deeper than the relationname got r for r lookup filteredrelation relationname filteredrelation condition renameprefixfromq filteredrelation relationname alias filteredrelation condition self filteredrelationsfilteredrelation alias filteredrelation def namestopathself names opts allowmanytrue failonmissingfalse path nameswithpath for pos name in enumeratenames curnameswithpath name if name pk and opts is not none name opts pk name field none filteredrelation none try if opts is none raise fielddoesnotexist field opts getfieldname except fielddoesnotexist if name in self annotationselect field self annotationselectname outputfield elif name in self filteredrelations and pos 0 filteredrelation self filteredrelationsname if lookupsep in filteredrelation relationname parts filteredrelation relationname splitlookupsep filteredrelationpath field self namestopath parts opts allowmany failonmissing path extendfilteredrelationpath 1 else field opts getfieldfilteredrelation relationname if field is not none fields that contain onetomany relations with a generic model like a genericforeignkey cannot generate reverse relations and therefore cannot be used for reverse querying if field isrelation and not field relatedmodel raise fielderror field r does not generate an automatic reverse relation and therefore cannot be used for reverse querying if it is a genericforeignkey consider adding a genericrelation name try model field model meta concretemodel except attributeerror queryset annotate may introduce fields that aren t attached to a model model none else we didn t find the current field so move position back one step pos 1 if pos 1 or failonmissing available sorted getfieldnamesfromoptsopts self annotationselect self filteredrelations raise fielderror cannot resolve keyword s into field choices are s name joinavailable break check if we need any joins for concrete inheritance cases the field lives in parent but we are currently in one of its children if opts is not none and model is not opts model pathtoparent opts getpathtoparentmodel if pathtoparent path extendpathtoparent curnameswithpath1 extendpathtoparent opts pathtoparent1 toopts if hasattrfield pathinfos if filteredrelation pathinfos field getpathinfofilteredrelation else pathinfos field pathinfos if not allowmany for innerpos p in enumeratepathinfos if p m2m curnameswithpath1 extendpathinfos0 innerpos 1 nameswithpath appendcurnameswithpath raise multijoinpos 1 nameswithpath last pathinfos1 path extendpathinfos finalfield last joinfield opts last toopts targets last targetfields curnameswithpath1 extendpathinfos nameswithpath appendcurnameswithpath else local nonrelational field finalfield field targets field if failonmissing and pos 1 lennames raise fielderror cannot resolve keyword r into field join on s not permitted namespos 1 name break return path finalfield targets namespos 1 def setupjoins self names opts alias canreusenone allowmanytrue joins alias the transform can t be applied yet as joins must be trimmed later to avoid making every caller of this method look up transforms directly compute transforms here and create a partial that converts fields to the appropriate wrapped version def finaltransformerfield alias if not self aliascols alias none return field getcolalias try resolving all the names as fields first if there s an error treat trailing names as lookups until a field can be resolved lastfieldexception none for pivot in rangelennames 0 1 try path finalfield targets rest self namestopath names pivot opts allowmany failonmissingtrue except fielderror as exc if pivot 1 the first item cannot be a lookup so it s safe to raise the field error here raise else lastfieldexception exc else the transforms are the remaining items that couldn t be resolved into fields transforms namespivot break for name in transforms def transformfield alias name previous try wrapped previousfield alias return self trytransformwrapped name except fielderror fielderror is raised if the transform doesn t exist if isinstancefinalfield field and lastfieldexception raise lastfieldexception else raise finaltransformer functools partial transform namename previousfinaltransformer finaltransformer hastransforms true then add the path to the query s joins note that we can t trim joins at this stage we will need the information about join type of the trimmed joins for join in path if join filteredrelation filteredrelation join filteredrelation clone tablealias filteredrelation alias else filteredrelation none tablealias none opts join toopts if join direct nullable self isnullablejoin joinfield else nullable true connection self joinclass opts dbtable alias tablealias inner join joinfield nullable filteredrelationfilteredrelation reuse canreuse if join m2m else none alias self joinconnection reusereuse joins appendalias return joininfofinalfield targets opts joins path finaltransformer def trimjoinsself targets joins path joins joins for pos info in enumeratereversedpath if lenjoins 1 or not info direct break if info filteredrelation break jointargets t column for t in info joinfield foreignrelatedfields curtargets t column for t in targets if not curtargets issubsetjointargets break targetsdict r1 column r0 for r in info joinfield relatedfields if r1 column in curtargets targets tupletargetsdictt column for t in targets self unrefaliasjoins pop return targets joins1 joins classmethod def gencolscls exprs includeexternalfalse resolverefstrue for expr in exprs if isinstanceexpr col yield expr elif includeexternal and callable getattrexpr getexternalcols none yield from expr getexternalcols elif hasattrexpr getsourceexpressions if not resolverefs and isinstanceexpr ref continue yield from cls gencols expr getsourceexpressions includeexternalincludeexternal resolverefsresolverefs classmethod def gencolaliasescls exprs yield from expr alias for expr in cls gencolsexprs def resolverefself name allowjoinstrue reusenone summarizefalse annotation self annotations getname if annotation is not none if not allowjoins for alias in self gencolaliasesannotation if isinstanceself aliasmapalias join raise fielderror joined field references are not permitted in this query if summarize summarize currently means we are doing an aggregate query which is executed as a wrapped subquery if any of the aggregate elements reference an existing annotation in that case we need to return a ref to the subquery s annotation if name not in self annotationselect raise fielderror cannot aggregate over the s alias use annotate to promote it name return refname self annotationselectname else return annotation else fieldlist name splitlookupsep annotation self annotations getfieldlist0 if annotation is not none for transform in fieldlist1 annotation self trytransformannotation transform return annotation joininfo self setupjoins fieldlist self getmeta self getinitialalias canreusereuse targets finalalias joinlist self trimjoins joininfo targets joininfo joins joininfo path if not allowjoins and lenjoinlist 1 raise fielderror joined field references are not permitted in this query if lentargets 1 raise fielderror referencing multicolumn fields with f objects isn t supported verify that the last lookup in name is a field or a transform transformfunction raises fielderror if not transform joininfo transformfunctiontargets0 finalalias if reuse is not none reuse updatejoinlist return transform def splitexcludeself filterexpr canreuse nameswithpath generate the inner query query self classself model query filteredrelations self filteredrelations filterlhs filterrhs filterexpr if isinstancefilterrhs outerref filterrhs outerreffilterrhs elif isinstancefilterrhs f filterrhs outerreffilterrhs name query addfilterfilterlhs filterrhs query clearorderingforcetrue try to have as simple as possible subquery trim leading joins from the subquery trimmedprefix containslouter query trimstartnameswithpath col query select0 selectfield col target alias col alias if alias in canreuse pk selectfield model meta pk need to add a restriction so that outer query s filters are in effect for the subquery too query bumpprefixself lookupclass selectfield getlookupexact note that the query select0 alias is different from alias due to bumpprefix above lookup lookupclasspk getcolquery select0 alias pk getcolalias query where addlookup and query externalaliasesalias true else lookupclass selectfield getlookupexact lookup lookupclasscol resolvedouterreftrimmedprefix query where addlookup and condition neededinner self buildfilterexistsquery if containslouter ornullcondition self buildfilter sisnull trimmedprefix true currentnegatedtrue branchnegatedtrue canreusecanreuse condition addornullcondition or note that the end result will be not exists innerq or outercol is null this might look crazy but due to how null works this seems to be correct if the is null check is removed then if outercol is null we will not match the row return condition neededinner def setemptyself self where addnothingnode and for query in self combinedqueries query setempty def isemptyself return anyisinstancec nothingnode for c in self where children def setlimitsself lownone highnone if high is not none if self highmark is not none self highmark minself highmark self lowmark high else self highmark self lowmark high if low is not none if self highmark is not none self lowmark minself highmark self lowmark low else self lowmark self lowmark low if self lowmark self highmark self setempty def clearlimitsself return true if adding filters to this instance is still possible typically this means no limits or offsets have been put on the results remove all fields from select clause self select self defaultcols false self selectrelated false self setextramask self setannotationmask def clearselectfieldsself self select self valuesselect def addselectcolself col name self select col self valuesselect name def setselectself cols self defaultcols false self select tuplecols def adddistinctfieldsself fieldnames self distinctfields fieldnames self distinct true def addfieldsself fieldnames allowm2mtrue alias self getinitialalias opts self getmeta try cols for name in fieldnames join promotion note we must not remove any rows here so if there is no existing joins use outer join joininfo self setupjoins name splitlookupsep opts alias allowmanyallowm2m targets finalalias joins self trimjoins joininfo targets joininfo joins joininfo path for target in targets cols appendjoininfo transformfunctiontarget finalalias if cols self setselectcols except multijoin raise fielderrorinvalid field name s name except fielderror if lookupsep in name for lookups spanning over relationships show the error from the model on which the lookup failed raise else names sorted getfieldnamesfromoptsopts self extra self annotationselect self filteredrelations raise fielderror cannot resolve keyword r into field choices are s name joinnames def addorderingself ordering errors for item in ordering if isinstanceitem str if item continue item item removeprefix if item in self annotations continue if self extra and item in self extra continue namestopath validates the lookup a descriptive fielderror will be raise if it s not self namestopathitem splitlookupsep self model meta elif not hasattritem resolveexpression errors appenditem if getattritem containsaggregate false raise fielderror using an aggregate in orderby without also including it in annotate is not allowed s item if errors raise fielderrorinvalid orderby arguments s errors if ordering self orderby ordering else self defaultordering false def clearorderingself forcefalse cleardefaulttrue if not force and self issliced or self distinctfields or self selectforupdate return self orderby self extraorderby if cleardefault self defaultordering false def setgroupbyself allowaliasestrue if allowaliases and self valuesselect if grouping by aliases is allowed assign selected value aliases by moving them to annotations groupbyannotations valuesselect for alias expr in zipself valuesselect self select if isinstanceexpr col valuesselectalias expr else groupbyannotationsalias expr self annotations groupbyannotations self annotations self appendannotationmaskgroupbyannotations self select tuplevaluesselect values self valuesselect tuplevaluesselect groupby listself select for alias annotation in self annotationselect items if not groupbycols annotation getgroupbycols continue if allowaliases and not annotation containsaggregate groupby appendrefalias annotation else groupby extendgroupbycols self groupby tuplegroupby def addselectrelatedself fields if isinstanceself selectrelated bool fielddict else fielddict self selectrelated for field in fields d fielddict for part in field splitlookupsep d d setdefaultpart self selectrelated fielddict def addextraself select selectparams where params tables orderby if select we need to pair any placeholder markers in the select dictionary with their parameters in selectparams so that subsequent updates to the select dictionary also adjust the parameters appropriately selectpairs if selectparams paramiter iterselectparams else paramiter iter for name entry in select items self checkaliasname entry strentry entryparams pos entry finds while pos 1 if pos 0 or entrypos 1 entryparams appendnextparamiter pos entry finds pos 2 selectpairsname entry entryparams self extra updateselectpairs if where or params self where addextrawherewhere params and if tables self extratables tupletables if orderby self extraorderby orderby def cleardeferredloadingself add the given list of model field names to the set of fields to exclude from loading from the database when automatic column selection is done add the new field names to any existing field names that are deferred or removed from any existing field names that are marked as the only ones for immediate loading fields on related models are stored in the literal doubleunderscore format so that we can use a set datastructure we do the foobar splitting and handling when computing the sql column names as part of getcolumns add to existing deferred names remove names from the set of any existing immediate load names add the given list of model field names to the set of fields to retrieve when the sql is executed immediate loading fields the field names replace any existing immediate loading field names if there are field names already specified for deferred loading remove those names from the new fieldnames before storing the new names for immediate loading that is immediate loading overrides any existing immediate values but respects existing deferrals remove any existing deferred names from the current set before setting the new names replace any existing immediate load field names set the mask of annotations that will be returned by the select if names is none self annotationselectmask none else self annotationselectmask listdict fromkeysnames self annotationselectcache none def appendannotationmaskself names if self annotationselectmask is not none self setannotationmaskself annotationselectmask names def setextramaskself names if names is none self extraselectmask none else self extraselectmask setnames self extraselectcache none def setvaluesself fields self selectrelated false self cleardeferredloading self clearselectfields self hasselectfields true if fields fieldnames extranames annotationnames if not self extra and not self annotations shortcut if there are no extra or annotations then the values clause must be just field names fieldnames listfields else self defaultcols false for f in fields if f in self extraselect extranames appendf elif f in self annotationselect annotationnames appendf elif f in self annotations raise fielderror fcannot select the f alias use annotate to promote it else call namestopath to ensure a fielderror including annotations about to be masked as valid choices if f is not resolvable if self annotationselect self namestopathf splitlookupsep self model meta fieldnames appendf self setextramaskextranames self setannotationmaskannotationnames selected frozensetfieldnames extranames annotationnames else fieldnames f attname for f in self model meta concretefields selected frozensetfieldnames selected annotations must be known before setting the group by clause if self groupby is true self addfields f attname for f in self model meta concretefields false disable group by aliases to avoid orphaning references to the select clause which is about to be cleared self setgroupbyallowaliasesfalse self clearselectfields elif self groupby resolve group by annotation references if they are not part of the selected fields anymore groupby for expr in self groupby if isinstanceexpr ref and expr refs not in selected expr self annotationsexpr refs groupby appendexpr self groupby tuplegroupby self valuesselect tuplefieldnames self addfieldsfieldnames true property def annotationselectself if self annotationselectcache is not none return self annotationselectcache elif not self annotations return elif self annotationselectmask is not none self annotationselectcache k self annotationsk for k in self annotationselectmask if k in self annotations return self annotationselectcache else return self annotations property def extraselectself if self extraselectcache is not none return self extraselectcache if not self extra return elif self extraselectmask is not none self extraselectcache k v for k v in self extra items if k in self extraselectmask return self extraselectcache else return self extra def trimstartself nameswithpath trim and operate only on tables that were generated for the lookup part of the query that is avoid trimming joins generated for f expressions the path joinfield is a rel lets get the other side s field build the filter prefix lets still see if we can trim the first join from the inner query that is self we can t do this for left joins because we would miss those rows that have nothing on the outer side inner joins from filtered relations because we would miss their filters todo it might be possible to trim more joins from the start of the inner query if it happens to have a longer join chain containing the values in selectfields lets punt this one for now the found starting point is likely a joinclass instead of a basetableclass reference but the first entry in the query s from clause must not be a join check if the given field should be treated as nullable some backends treat as null and django treats such fields as nullable for those backends in such situations field null can be false even if we should treat the field as nullable we need to use defaultdbalias here as queryset does not have nor should it have knowledge of which connection is going to be used the proper fix would be to defer all decisions where isnullable is needed to the compiler stage but that is not easy to do currently return the field name and direction for an order specification for example foo is returned as foo desc the default param is used to indicate which way no prefix or a prefix should sort the prefix always sorts the opposite way a class to abstract away join promotion problems for complex filter conditions maps of table alias to how many times it is seen as required for inner andor outer joins add single vote per item to self votes parameter can be any iterable change join types so that the generated query is as efficient as possible but still correct so change as many joins as possible to inner but don t make outer joins inner if that could remove results from the query the effectiveconnector is used so that not a and b is treated similarly to a or b for join promotion we must use outer joins in or case when the join isn t contained in all of the joins otherwise the inner join itself could remove valid results consider the case where a model with rela and relb relations is queried with relacol1 relbcol2 now if rela join doesn t produce any results is null for example reverse foreign key or null value in direct foreign key and there is a matching row in relb with col2 then an inner join to rela would remove a valid match from the query so we need to promote any existing inner to louter it is possible this promotion in turn will be demoted later on if connector is and and there is a filter that can match only when there is a joinable row then use inner for example in relacol1 relbcol2 if either of the rels produce null as join output then the col1 or col2 can t match as nullanything is always false for the or case if all children voted for a join to be inner then we can use inner for the join for example relacolicontainsalex relacolicontainsrussell then if rela doesn t produce any rows the whole condition can t match hence we can safely use inner join finally what happens in cases where we have relacol1relbcol2 relacolgte0 now we first generate the or clause and promote joins for it in the first if branch above both rela and relb are promoted to louter joins after that we do the and case the or case voted no inner joins but the relacolgte0 votes inner join for rela we demote it back to inner join in and case a single vote is enough the demotion is ok if rela doesn t produce rows then the relacolgte0 clause can t be true and thus the whole clause must be false so it is safe to use inner join note that in this example we could just as well have the gte clause and the or clause swapped or we could replace the gte clause with an or clause containing relacol1relacol2 and again we could safely demote to inner quotation marks whitespace characters semicolons or inline sql comments are forbidden in column aliases inspired from https www postgresql org docs current sql syntax lexical html sql syntax identifiers a single raw sql query mirror some properties of a normal query so that the compiler can be used to process results used for offset limit always execute a new query for a new iterator this could be optimized with a cache at the expense of ram if the database can t use chunked reads we need to make sure we evaluate the entire query up front adapt parameters to the database as much as possible considering that the target type isn t known see 17755 a single sql query sql related attributes select and related select clauses are expressions to use in the select clause of the query the select is used for cases where we want to set up the select clause to contain other than default fields values subqueries note that annotations go to annotations dictionary the group_by attribute can have one of the following forms none no group by at all in the query a tuple of expressions group by at least those expressions string refs are also allowed for now true group by all select fields of the model see compiler get_group_by for details used for offset limit used for offset limit arbitrary limit for select_related to prevents infinite recursion holds the selects defined by a call to values or values_list excluding annotation_select and extra_select sql annotation related attributes set combination attributes these are for extensions the contents are more or less appended verbatim to the appropriate clause a tuple that is a set of model field names and either true if these are the fields to defer or false if these are the only fields to load alias_map is the most important data structure regarding joins it s used for recording which joins exist in the query and what types they are the key is the alias of the joined table possibly the table name and the value is a join like object see sql datastructures join for more information whether to provide alias to columns during reference resolving sometimes the query contains references to aliases in outer queries as a result of split_exclude correct alias quoting needs to know these aliases too map external tables to whether they are aliased maps table names to list of aliases maps alias annotation expression these are for extensions the contents are more or less appended verbatim to the appropriate clause maps col_alias col_sql params return the query as a string of sql with the parameter values substituted in use sql_with_params to see the unsubstituted string parameter values won t necessarily be quoted correctly since that is done by the database interface at execution time return the query as an sql string and the parameters that will be substituted into the query limit the amount of work when a query is deepcopied return the options instance the model _meta from which to start processing normally this is self model _meta but it can be changed by subclasses return a copy of the current query a lightweight alternative to deepcopy copy references to everything clone attributes that can t use shallow copy _annotation_select_cache cannot be copied as doing so breaks the necessary state in which both annotations and _annotation_select_cache point to the same underlying objects it will get re populated in the cloned queryset the next time it s used use deepcopy because select_related stores fields in nested dicts clear the cached_property if it exists return a copy of the current query that s ready for another operation the klass argument changes the type of the query e g updatequery return the dictionary with the values of the existing aggregations store annotation mask prior to temporarily adding aggregations for resolving purpose to facilitate their subsequent removal temporarily add aggregate to annotations to allow remaining members of aggregates to resolve against each others stash resolved aggregates now that they have been allowed to resolve against each other existing usage of aggregation can be determined by the presence of selected aggregates but also by filters against aliased aggregates decide if we need to use a subquery existing aggregations would cause incorrect results as get_aggregation must produce just one result and thus must not use group by if the query has limit or distinct or uses set operations then those operations must be done in a subquery so that the query aggregates on the limit and or distinct results instead of applying the distinct and limit after the aggregation queries with distinct_fields need ordering and when a limit is applied we must take the slice from the ordered query otherwise no need for ordering if the inner query uses default select and it has some aggregate annotations then we must make sure the inner query is grouped by the main model s primary key however clearing the select clause can alter results if distinct is used mask existing annotations that are not referenced by aggregates to be pushed to the outer query unless filtering against window functions is involved as it requires complex realising avoid eliding expressions that might have an incidence on the implicit grouping logic add aggregates to the outer aggregatequery this requires making sure all columns referenced by the aggregates are selected in the inner query it is achieved by retrieving all column references by the aggregates explicitly selecting them in the inner query and making sure the aggregates are repointed to them in case of model objects 0 3 count there would be no field selected in the inner query yet we must use a subquery so make sure at least one field is selected inline reference to existing annotations and mask them as they are unnecessary given only the summarized aggregations are requested perform a count query using the current filter constraints disable group by aliases to avoid orphaning references to the select clause which is about to be cleared merge the rhs query into the current one with any rhs effects being applied after that is to the right of anything in the current query rhs is not modified during a call to this function the connector parameter describes how to connect filters from the rhs query if lhs and rhs shares the same alias prefix it is possible to have conflicting alias changes like t4 t5 t5 t6 which might end up as t4 t6 while combining two querysets to prevent this change an alias prefix of the rhs and update current aliases accordingly except if the alias is the base table since it must be present in the query on both sides work out how to relabel the rhs aliases if necessary determine which existing joins can be reused when combining the query with and we must recreate all joins for m2m filters when combining with or we can reuse joins the reason is that in and case a single row can t fulfill a condition like revrel__col 1 revrel__col 2 but there might be two different related rows matching this condition in or case a single true is enough so single row is enough too note that we will be creating duplicate joins for non m2m joins in the and case the results will be correct but this creates too many joins this is something that could be fixed later on now add the joins from rhs query into the new query skipping base table if the left side of the join was already relabeled use the updated alias we can t reuse the same join again in the query if we have two distinct joins for the same connection in rhs query then the combined query must have two joins too the alias was unused in the rhs query unref it so that it will be unused in the new query too we have to add and unref the alias so that join promotion has information of the join type for the unused alias combine subqueries aliases to ensure aliases relabelling properly handle subqueries when combining where and select clauses now relabel a copy of the rhs where clause and add it to the current one selection columns and extra extensions are those provided by rhs it would be nice to be able to handle this but the queries don t really make sense or return consistent value sets not worth the extra complexity when you can write a real query instead ordering uses the rhs ordering unless it has none in which case the current ordering is used all concrete fields that are not part of the defer mask must be loaded if a relational field is encountered it gets added to the mask for it be considered if select_related and the cycle continues by recursively calling this function remaining defer entries must be references to reverse relationships the following code is expected to raise fielderror if it encounters a malformed defer entry while virtual fields such as many to many and generic foreign keys cannot be effectively deferred we ve historically allowed them to be passed to queryset defer ignore such field references until a layer of validation at mask alteration time will be implemented eventually only include fields mentioned in the mask retrieve the actual field associated with reverse relationships as that s what is expected in the select mask convert the self deferred_loading data structure to an alternate data structure describing the field that will be loaded this is used to compute the columns to select from the database and also by the queryset class to work out which fields are being initialized on each model models that have all their fields included aren t mentioned in the result only those that have field restrictions in place return a table alias for the given table_name and whether this is a new alias or not if create is true a new alias is always created otherwise the most recently created alias for the table if one exists is reused create a new alias for this table the first occurrence of a table uses the table name directly increases the reference count for this alias decreases the reference count for this alias promote recursively the join type of given aliases and its children to an outer join if unconditional is false only promote the join if it is nullable or the parent join is an outer join the children promotion is done to avoid join chains that contain a louter b inner c so if we have currently a inner b inner c and a b is promoted then we must also promote b c automatically or otherwise the promotion of a b doesn t actually change anything in the query results this is the base table first from entry this table isn t really joined at all in the query so we should not alter its join type only the first alias skipped above should have none join_type join type of alias changed so re examine all aliases that refer to this one change join type from louter to inner for all joins in aliases similarly to promote_joins this method must ensure no join chains containing first an outer then an inner join are generated if we are demoting b c join in chain a louter b louter c then we must demote a b automatically or otherwise the demotion of b c doesn t actually change anything in the query results reset reference counts for aliases so that they match the value passed in to_counts change the aliases in change_map which maps old alias new alias relabelling any references to them in select columns and the where clause if keys and values of change_map were to intersect an alias might be updated twice e g t4 t5 t5 t6 so also t4 t6 depending on their order in change_map 1 update references in select normal columns plus aliases group by and where 2 rename the alias in the internal table alias datastructures table is aliased or it s being changed and thus is aliased change the alias prefix to the next letter in the alphabet in a way that the other query s aliases and this query s aliases will not conflict even tables that previously had no alias will get an alias after this call to prevent changing aliases use the exclude parameter generate a sequence of characters in alphabetical order a b c when the alphabet is finished the sequence will continue with the cartesian product aa ab ac no clashes between self and outer query should be possible explicitly avoid infinite loop the constant divider is based on how much depth recursive subquery references add to the stack this value might need to be adjusted when adding or removing function calls from the code path in charge of performing these operations return the first alias for this query after increasing its reference count return the number of tables in this query with a non zero reference count after execution the reference counts are zeroed so tables added in compiler will not be seen by this method return an alias for the join either reusing an existing alias for that join or creating a new one join is either a base_table_class or join_class the reuse parameter can be either none which means all joins are reusable or it can be a set containing the aliases that can be reused a join is always created as louter if the lhs alias is louter to make sure chains like t1 louter t2 inner t3 aren t generated all new joins are created as louter if the join is nullable reuse the most recent alias of the joined table a many to many relation may be joined multiple times no reuse is possible so we need a new alias some joins were during expression resolving they must be present before the one we just added make sure the given model is joined in the query if model isn t a parent of opts or if it is none this method is a no op the alias is the root alias for starting the join seen is a dict of model alias of existing joins it must also contain a mapping of none some alias this will be returned in the no op case proxy model have elements in base chain with no parents assign the new options object and skip to the next base in that case add a single annotation expression to the query subqueries need to use a different set of aliases than the outer query resolve combined queries outer query s aliases are considered external if wrapper is referenced by an alias for an explicit group by through values a reference to this expression and not the self must be returned to ensure external column references are not grouped against as well some backends e g oracle raise an error when a subquery contains unnecessary order by clause the items of the iterable may be expressions and therefore need to be resolved independently namedtuple solve the lookup type from the lookup e g foobar__id__icontains check whether the object passed while querying is of the correct type if not raise a valueerror specifying the wrong object check the type of object passed to query relations check that the field and the queryset use the same model in a query like filter objects all for example the opts would be s from the field and value model would be objects all queryset s model also the field is the related field on the lhs side raise an error if expression cannot be used in a where clause try to extract transforms and lookup from given lhs the lhs value is something that works like sqlexpression the rhs value is what the lookup is going to compare against the lookups is a list of names to extract using get_lookup and get_transform __exact is the default lookup if one isn t given first try get_lookup so that the lookup takes precedence if the lhs supports both transform and lookup for the name a lookup wasn t found try to interpret the name as a transform and do an exact lookup against it interpret __exact none as the sql is null otherwise reject all uses of none as a query value unless the lookup supports it for oracle is equivalent to null the check must be done at this stage because join promotion can t be done in the compiler using default_db_alias isn t nice but it s the best that can be done here a similar thing is done in is_nullable too helper method for build_lookup try to fetch and initialize a transform for name parameter from lhs build a wherenode for a single filter clause but don t add it to this query query add_q will then add this filter to the where node the branch_negated tells us if the current branch contains any negations this will be used to determine if subqueries are needed the current_negated is used to determine if the current filter is negated or not and this will be used to determine if is null filtering is needed the difference between current_negated and branch_negated is that branch_negated is set on first negation but current_negated is flipped for each negation note that add_filter will not do any negating itself that is done upper in the code by add_q the can_reuse is a set of reusable joins for multijoins the method will create a filter clause that can be added to the current query however if the filter isn t added to the query then the caller is responsible for unreffing the joins used prevent iterator from being consumed by check_related_objects split_exclude needs to know which joins were generated for the lookup parts update used_joins before trimming since they are reused to determine which joins could be later promoted to inner the condition added here will be sql like this not col is not null where the first not is added in upper layers of code the reason for addition is that if col is null then col someval will result in sql unknown which isn t the same as in python the python none handling is wanted and it can be gotten by col is null or col someval not col is not null and col someval if someval is a nullable column someval is not null is added a preprocessor for the internal _add_q responsible for doing final join promotion for join promotion this case is doing an and for the added q_object and existing conditions so any existing inner join forces the join type to remain inner existing outer joins can however be demoted consider case where rel_a is louter and rel_a__col 1 is added if rel_a doesn t produce any rows then the whole condition must fail so demotion is ok add a q object to the current filter walk the list of names and turns them into pathinfo tuples a single name in names can generate multiple pathinfos m2m for example names is the path of names to travel opts is the model options we start the name resolving from allow_many is as for setup_joins if fail_on_missing is set to true then a name that can t be resolved will generate a fielderror return a list of pathinfo tuples in addition return the final field the last used join field and target which is a field guaranteed to contain the same value as the final field finally return those names that weren t found which are likely transforms and the final lookup fields that contain one to many relations with a generic model like a genericforeignkey cannot generate reverse relations and therefore cannot be used for reverse querying queryset annotate may introduce fields that aren t attached to a model we didn t find the current field so move position back one step check if we need any joins for concrete inheritance cases the field lives in parent but we are currently in one of its children local non relational field compute the necessary table joins for the passage through the fields given in names opts is the options class for the current model which gives the table we are starting from alias is the alias for the table to start the joining from the can_reuse defines the reverse foreign key joins we can reuse it can be none in which case all joins are reusable or a set of aliases that can be reused note that non reverse foreign keys are always reusable when using setup_joins if allow_many is false then any reverse foreign key seen will generate a multijoin exception return the final field involved in the joins the target field used for any where constraint the final opts value the joins the field path traveled to generate the joins and a transform function that takes a field and alias and is equivalent to field get_col alias in the simple case but wraps field transforms if they were included in names the target field is the field containing the concrete value final field can be something different for example foreign key pointing to that value final field is needed for example in some value conversions convert obj in fk__id obj to pk val using the foreign key field for example the transform can t be applied yet as joins must be trimmed later to avoid making every caller of this method look up transforms directly compute transforms here and create a partial that converts fields to the appropriate wrapped version try resolving all the names as fields first if there s an error treat trailing names as lookups until a field can be resolved the first item cannot be a lookup so it s safe to raise the field error here the transforms are the remaining items that couldn t be resolved into fields fielderror is raised if the transform doesn t exist then add the path to the query s joins note that we can t trim joins at this stage we will need the information about join type of the trimmed joins the target parameter is the final field being joined to joins is the full list of join aliases the path contain the pathinfos used to create the joins return the final target field and table alias and the new active joins always trim any direct join if the target column is already in the previous table can t trim reverse joins as it s unknown if there s anything on the other side of the join summarize currently means we are doing an aggregate query which is executed as a wrapped subquery if any of the aggregate elements reference an existing annotation in that case we need to return a ref to the subquery s annotation verify that the last lookup in name is a field or a transform transform_function raises fielderror if not when doing an exclude against any kind of n to many relation we need to use a subquery this method constructs the nested query given the original exclude filter filter_expr and the portion up to the first n to many relation field for example if the origin filter is q child__name foo filter_expr is child__name foo and can_reuse is a set of joins usable for filters in the original query we will turn this into equivalent of where not exists select 1 from child where name foo and child parent_id parent id limit 1 generate the inner query try to have as simple as possible subquery trim leading joins from the subquery need to add a restriction so that outer query s filters are in effect for the subquery too note that the query select 0 alias is different from alias due to bump_prefix above note that the end result will be not exists inner_q or outercol is null this might look crazy but due to how null works this seems to be correct if the is null check is removed then if outercol is null we will not match the row adjust the limits on the rows retrieved use low high to set these as it makes it more pythonic to read and write when the sql query is created convert them to the appropriate offset and limit values apply any limits passed in here to the existing constraints add low to the current low value and clamp both to any existing high value clear any existing limits return true if adding filters to this instance is still possible typically this means no limits or offsets have been put on the results remove all fields from select clause clear the list of fields to select but not extra_select columns some queryset types completely replace any existing list of select columns add and resolve the given fields to the query s distinct on clause add the given model fields to the select set add the field names in the order specified join promotion note we must not remove any rows here so if there is no existing joins use outer join for lookups spanning over relationships show the error from the model on which the lookup failed add items from the ordering sequence to the query s order by clause these items are either field names not column names possibly with a direction prefix or or orderby expressions if ordering is empty clear all ordering from the query names_to_path validates the lookup a descriptive fielderror will be raise if it s not remove any ordering settings if the current query allows it without side effects set force to true to clear the ordering regardless if clear_default is true there will be no ordering in the resulting query not even the model s default expand the group by clause required by the query this will usually be the set of all non aggregate fields in the return data if the database backend supports grouping by the primary key and the query would be equivalent the optimization will be made automatically if grouping by aliases is allowed assign selected value aliases by moving them to annotations set up the select_related data structure so that we only select certain related models as opposed to all models when self select_related true add data to the various extra_ attributes for user created additions to the query we need to pair any placeholder markers in the select dictionary with their parameters in select_params so that subsequent updates to the select dictionary also adjust the parameters appropriately remove any fields from the deferred loading set add the given list of model field names to the set of fields to exclude from loading from the database when automatic column selection is done add the new field names to any existing field names that are deferred or removed from any existing field names that are marked as the only ones for immediate loading fields on related models are stored in the literal double underscore format so that we can use a set datastructure we do the foo__bar splitting and handling when computing the sql column names as part of get_columns add to existing deferred names remove names from the set of any existing immediate load names add the given list of model field names to the set of fields to retrieve when the sql is executed immediate loading fields the field names replace any existing immediate loading field names if there are field names already specified for deferred loading remove those names from the new field_names before storing the new names for immediate loading that is immediate loading overrides any existing immediate values but respects existing deferrals remove any existing deferred names from the current set before setting the new names replace any existing immediate load field names set the mask of annotations that will be returned by the select set the mask of extra select items that will be returned by select don t remove them from the query since they might be used later shortcut if there are no extra or annotations then the values clause must be just field names call names_to_path to ensure a fielderror including annotations about to be masked as valid choices if f is not resolvable selected annotations must be known before setting the group by clause disable group by aliases to avoid orphaning references to the select clause which is about to be cleared resolve group by annotation references if they are not part of the selected fields anymore return the dictionary of aggregate columns that are not masked and should be used in the select clause cache this result for performance trim joins from the start of the join path the candidates for trim are the pathinfos in names_with_path structure that are m2m joins also set the select column so the start matches the join this method is meant to be used for generating the subquery joins cols in split_exclude return a lookup usable for doing outerq filter lookup self and a boolean indicating if the joins in the prefix contain a left outer join _ trim and operate only on tables that were generated for the lookup part of the query that is avoid trimming joins generated for f expressions the path join_field is a rel lets get the other side s field build the filter prefix lets still see if we can trim the first join from the inner query that is self we can t do this for left joins because we would miss those rows that have nothing on the outer side inner joins from filtered relations because we would miss their filters todo it might be possible to trim more joins from the start of the inner query if it happens to have a longer join chain containing the values in select_fields lets punt this one for now the found starting point is likely a join_class instead of a base_table_class reference but the first entry in the query s from clause must not be a join check if the given field should be treated as nullable some backends treat as null and django treats such fields as nullable for those backends in such situations field null can be false even if we should treat the field as nullable we need to use default_db_alias here as queryset does not have nor should it have knowledge of which connection is going to be used the proper fix would be to defer all decisions where is_nullable is needed to the compiler stage but that is not easy to do currently return the field name and direction for an order specification for example foo is returned as foo desc the default param is used to indicate which way no prefix or a prefix should sort the prefix always sorts the opposite way a class to abstract away join promotion problems for complex filter conditions maps of table alias to how many times it is seen as required for inner and or outer joins add single vote per item to self votes parameter can be any iterable change join types so that the generated query is as efficient as possible but still correct so change as many joins as possible to inner but don t make outer joins inner if that could remove results from the query the effective_connector is used so that not a and b is treated similarly to a or b for join promotion we must use outer joins in or case when the join isn t contained in all of the joins otherwise the inner join itself could remove valid results consider the case where a model with rel_a and rel_b relations is queried with rel_a__col 1 rel_b__col 2 now if rel_a join doesn t produce any results is null for example reverse foreign key or null value in direct foreign key and there is a matching row in rel_b with col 2 then an inner join to rel_a would remove a valid match from the query so we need to promote any existing inner to louter it is possible this promotion in turn will be demoted later on if connector is and and there is a filter that can match only when there is a joinable row then use inner for example in rel_a__col 1 rel_b__col 2 if either of the rels produce null as join output then the col 1 or col 2 can t match as null anything is always false for the or case if all children voted for a join to be inner then we can use inner for the join for example rel_a__col__icontains alex rel_a__col__icontains russell then if rel_a doesn t produce any rows the whole condition can t match hence we can safely use inner join finally what happens in cases where we have rel_a__col 1 rel_b__col 2 rel_a__col__gte 0 now we first generate the or clause and promote joins for it in the first if branch above both rel_a and rel_b are promoted to louter joins after that we do the and case the or case voted no inner joins but the rel_a__col__gte 0 votes inner join for rel_a we demote it back to inner join in and case a single vote is enough the demotion is ok if rel_a doesn t produce rows then the rel_a__col__gte 0 clause can t be true and thus the whole clause must be false so it is safe to use inner join note that in this example we could just as well have the __gte clause and the or clause swapped or we could replace the __gte clause with an or clause containing rel_a__col 1 rel_a__col 2 and again we could safely demote to inner
import copy
import difflib
import functools
import sys
from collections import Counter, namedtuple
from collections.abc import Iterator, Mapping
from itertools import chain, count, product
from string import ascii_uppercase

from django.core.exceptions import FieldDoesNotExist, FieldError
from django.db import DEFAULT_DB_ALIAS, NotSupportedError, connections
from django.db.models.aggregates import Count
from django.db.models.constants import LOOKUP_SEP
from django.db.models.expressions import (
    BaseExpression,
    Col,
    Exists,
    F,
    OuterRef,
    Ref,
    ResolvedOuterRef,
    Value,
)
from django.db.models.fields import Field
from django.db.models.fields.related_lookups import MultiColSource
from django.db.models.lookups import Lookup
from django.db.models.query_utils import (
    Q,
    check_rel_lookup_compatibility,
    refs_expression,
)
from django.db.models.sql.constants import INNER, LOUTER, ORDER_DIR, SINGLE
from django.db.models.sql.datastructures import BaseTable, Empty, Join, MultiJoin
from django.db.models.sql.where import AND, OR, ExtraWhere, NothingNode, WhereNode
from django.utils.functional import cached_property
from django.utils.regex_helper import _lazy_re_compile
from django.utils.tree import Node

__all__ = ["Query", "RawQuery"]



FORBIDDEN_ALIAS_PATTERN = _lazy_re_compile(r"['`\"\]\[;\s]|--|/\*|\*/")



EXPLAIN_OPTIONS_PATTERN = _lazy_re_compile(r"[\w-]+")


def get_field_names_from_opts(opts):
    if opts is None:
        return set()
    return set(
        chain.from_iterable(
            (f.name, f.attname) if f.concrete else (f.name,) for f in opts.get_fields()
        )
    )


def get_paths_from_expression(expr):
    if isinstance(expr, F):
        yield expr.name
    elif hasattr(expr, "flatten"):
        for child in expr.flatten():
            if isinstance(child, F):
                yield child.name
            elif isinstance(child, Q):
                yield from get_children_from_q(child)


def get_children_from_q(q):
    for child in q.children:
        if isinstance(child, Node):
            yield from get_children_from_q(child)
        elif isinstance(child, tuple):
            lhs, rhs = child
            yield lhs
            if hasattr(rhs, "resolve_expression"):
                yield from get_paths_from_expression(rhs)
        elif hasattr(child, "resolve_expression"):
            yield from get_paths_from_expression(child)


def get_child_with_renamed_prefix(prefix, replacement, child):
    if isinstance(child, Node):
        return rename_prefix_from_q(prefix, replacement, child)
    if isinstance(child, tuple):
        lhs, rhs = child
        lhs = lhs.replace(prefix, replacement, 1)
        if not isinstance(rhs, F) and hasattr(rhs, "resolve_expression"):
            rhs = get_child_with_renamed_prefix(prefix, replacement, rhs)
        return lhs, rhs

    if isinstance(child, F):
        child = child.copy()
        child.name = child.name.replace(prefix, replacement, 1)
    elif hasattr(child, "resolve_expression"):
        child = child.copy()
        child.set_source_expressions(
            [
                get_child_with_renamed_prefix(prefix, replacement, grand_child)
                for grand_child in child.get_source_expressions()
            ]
        )
    return child


def rename_prefix_from_q(prefix, replacement, q):
    return Q.create(
        [get_child_with_renamed_prefix(prefix, replacement, c) for c in q.children],
        q.connector,
        q.negated,
    )


JoinInfo = namedtuple(
    "JoinInfo",
    ("final_field", "targets", "opts", "joins", "path", "transform_function"),
)


class RawQuery:
    

    def __init__(self, sql, using, params=()):
        self.params = params
        self.sql = sql
        self.using = using
        self.cursor = None

        
        
        self.low_mark, self.high_mark = 0, None  
        self.extra_select = {}
        self.annotation_select = {}

    def chain(self, using):
        return self.clone(using)

    def clone(self, using):
        return RawQuery(self.sql, using, params=self.params)

    def get_columns(self):
        if self.cursor is None:
            self._execute_query()
        converter = connections[self.using].introspection.identifier_converter
        return [converter(column_meta[0]) for column_meta in self.cursor.description]

    def __iter__(self):
        
        
        self._execute_query()
        if not connections[self.using].features.can_use_chunked_reads:
            
            
            result = list(self.cursor)
        else:
            result = self.cursor
        return iter(result)

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__, self)

    @property
    def params_type(self):
        if self.params is None:
            return None
        return dict if isinstance(self.params, Mapping) else tuple

    def __str__(self):
        if self.params_type is None:
            return self.sql
        return self.sql % self.params_type(self.params)

    def _execute_query(self):
        connection = connections[self.using]

        
        
        params_type = self.params_type
        adapter = connection.ops.adapt_unknown_value
        if params_type is tuple:
            params = tuple(adapter(val) for val in self.params)
        elif params_type is dict:
            params = {key: adapter(val) for key, val in self.params.items()}
        elif params_type is None:
            params = None
        else:
            raise RuntimeError("Unexpected params type: %s" % params_type)

        self.cursor = connection.cursor()
        self.cursor.execute(self.sql, params)


ExplainInfo = namedtuple("ExplainInfo", ("format", "options"))


class Query(BaseExpression):
    

    alias_prefix = "T"
    empty_result_set_value = None
    subq_aliases = frozenset([alias_prefix])

    compiler = "SQLCompiler"

    base_table_class = BaseTable
    join_class = Join

    default_cols = True
    default_ordering = True
    standard_ordering = True

    filter_is_sticky = False
    subquery = False

    
    
    
    
    
    select = ()
    
    
    
    
    
    
    group_by = None
    order_by = ()
    low_mark = 0  
    high_mark = None  
    distinct = False
    distinct_fields = ()
    select_for_update = False
    select_for_update_nowait = False
    select_for_update_skip_locked = False
    select_for_update_of = ()
    select_for_no_key_update = False
    select_related = False
    has_select_fields = False
    
    max_depth = 5
    
    
    values_select = ()

    
    annotation_select_mask = None
    _annotation_select_cache = None

    
    combinator = None
    combinator_all = False
    combined_queries = ()

    
    
    extra_select_mask = None
    _extra_select_cache = None

    extra_tables = ()
    extra_order_by = ()

    
    
    deferred_loading = (frozenset(), True)

    explain_info = None

    def __init__(self, model, alias_cols=True):
        self.model = model
        self.alias_refcount = {}
        
        
        
        
        
        self.alias_map = {}
        
        self.alias_cols = alias_cols
        
        
        
        
        self.external_aliases = {}
        self.table_map = {}  
        self.used_aliases = set()

        self.where = WhereNode()
        
        self.annotations = {}
        
        
        self.extra = {}  

        self._filtered_relations = {}

    @property
    def output_field(self):
        if len(self.select) == 1:
            select = self.select[0]
            return getattr(select, "target", None) or select.field
        elif len(self.annotation_select) == 1:
            return next(iter(self.annotation_select.values())).output_field

    @cached_property
    def base_table(self):
        for alias in self.alias_map:
            return alias

    def __str__(self):
        
        sql, params = self.sql_with_params()
        return sql % params

    def sql_with_params(self):
        
        return self.get_compiler(DEFAULT_DB_ALIAS).as_sql()

    def __deepcopy__(self, memo):
        
        result = self.clone()
        memo[id(self)] = result
        return result

    def get_compiler(self, using=None, connection=None, elide_empty=True):
        if using is None and connection is None:
            raise ValueError("Need either using or connection")
        if using:
            connection = connections[using]
        return connection.ops.compiler(self.compiler)(
            self, connection, using, elide_empty
        )

    def get_meta(self):
        
        if self.model:
            return self.model._meta

    def clone(self):
        
        obj = Empty()
        obj.__class__ = self.__class__
        
        obj.__dict__ = self.__dict__.copy()
        
        obj.alias_refcount = self.alias_refcount.copy()
        obj.alias_map = self.alias_map.copy()
        obj.external_aliases = self.external_aliases.copy()
        obj.table_map = self.table_map.copy()
        obj.where = self.where.clone()
        obj.annotations = self.annotations.copy()
        if self.annotation_select_mask is not None:
            obj.annotation_select_mask = self.annotation_select_mask.copy()
        if self.combined_queries:
            obj.combined_queries = tuple(
                [query.clone() for query in self.combined_queries]
            )
        
        
        
        
        
        obj._annotation_select_cache = None
        obj.extra = self.extra.copy()
        if self.extra_select_mask is not None:
            obj.extra_select_mask = self.extra_select_mask.copy()
        if self._extra_select_cache is not None:
            obj._extra_select_cache = self._extra_select_cache.copy()
        if self.select_related is not False:
            
            
            obj.select_related = copy.deepcopy(obj.select_related)
        if "subq_aliases" in self.__dict__:
            obj.subq_aliases = self.subq_aliases.copy()
        obj.used_aliases = self.used_aliases.copy()
        obj._filtered_relations = self._filtered_relations.copy()
        
        obj.__dict__.pop("base_table", None)
        return obj

    def chain(self, klass=None):
        
        obj = self.clone()
        if klass and obj.__class__ != klass:
            obj.__class__ = klass
        if not obj.filter_is_sticky:
            obj.used_aliases = set()
        obj.filter_is_sticky = False
        if hasattr(obj, "_setup_query"):
            obj._setup_query()
        return obj

    def relabeled_clone(self, change_map):
        clone = self.clone()
        clone.change_aliases(change_map)
        return clone

    def _get_col(self, target, field, alias):
        if not self.alias_cols:
            alias = None
        return target.get_col(alias, field)

    def get_aggregation(self, using, aggregate_exprs):
        
        if not aggregate_exprs:
            return {}
        
        
        refs_subquery = False
        refs_window = False
        replacements = {}
        annotation_select_mask = self.annotation_select_mask
        for alias, aggregate_expr in aggregate_exprs.items():
            self.check_alias(alias)
            aggregate = aggregate_expr.resolve_expression(
                self, allow_joins=True, reuse=None, summarize=True
            )
            if not aggregate.contains_aggregate:
                raise TypeError("%s is not an aggregate expression" % alias)
            
            
            self.append_annotation_mask([alias])
            aggregate_refs = aggregate.get_refs()
            refs_subquery |= any(
                getattr(self.annotations[ref], "contains_subquery", False)
                for ref in aggregate_refs
            )
            refs_window |= any(
                getattr(self.annotations[ref], "contains_over_clause", True)
                for ref in aggregate_refs
            )
            aggregate = aggregate.replace_expressions(replacements)
            self.annotations[alias] = aggregate
            replacements[Ref(alias, aggregate)] = aggregate
        
        
        aggregates = {alias: self.annotations.pop(alias) for alias in aggregate_exprs}
        self.set_annotation_mask(annotation_select_mask)
        
        
        _, having, qualify = self.where.split_having_qualify()
        has_existing_aggregation = (
            any(
                getattr(annotation, "contains_aggregate", True)
                for annotation in self.annotations.values()
            )
            or having
        )
        
        
        
        
        
        
        
        
        
        
        if (
            isinstance(self.group_by, tuple)
            or self.is_sliced
            or has_existing_aggregation
            or refs_subquery
            or refs_window
            or qualify
            or self.distinct
            or self.combinator
        ):
            from django.db.models.sql.subqueries import AggregateQuery

            inner_query = self.clone()
            inner_query.subquery = True
            outer_query = AggregateQuery(self.model, inner_query)
            inner_query.select_for_update = False
            inner_query.select_related = False
            inner_query.set_annotation_mask(self.annotation_select)
            
            
            
            inner_query.clear_ordering(force=False)
            if not inner_query.distinct:
                
                
                
                
                
                if inner_query.default_cols and has_existing_aggregation:
                    inner_query.group_by = (
                        self.model._meta.pk.get_col(inner_query.get_initial_alias()),
                    )
                inner_query.default_cols = False
                if not qualify:
                    
                    
                    
                    
                    annotation_mask = set()
                    if isinstance(self.group_by, tuple):
                        for expr in self.group_by:
                            annotation_mask |= expr.get_refs()
                    for aggregate in aggregates.values():
                        annotation_mask |= aggregate.get_refs()
                    
                    
                    for annotation_alias, annotation in self.annotation_select.items():
                        if annotation.get_group_by_cols():
                            annotation_mask.add(annotation_alias)
                    inner_query.set_annotation_mask(annotation_mask)

            
            
            
            
            
            col_refs = {}
            for alias, aggregate in aggregates.items():
                replacements = {}
                for col in self._gen_cols([aggregate], resolve_refs=False):
                    if not (col_ref := col_refs.get(col)):
                        index = len(col_refs) + 1
                        col_alias = f"__col{index}"
                        col_ref = Ref(col_alias, col)
                        col_refs[col] = col_ref
                        inner_query.annotations[col_alias] = col
                        inner_query.append_annotation_mask([col_alias])
                    replacements[col] = col_ref
                outer_query.annotations[alias] = aggregate.replace_expressions(
                    replacements
                )
            if (
                inner_query.select == ()
                and not inner_query.default_cols
                and not inner_query.annotation_select_mask
            ):
                
                
                
                inner_query.select = (
                    self.model._meta.pk.get_col(inner_query.get_initial_alias()),
                )
        else:
            outer_query = self
            self.select = ()
            self.default_cols = False
            self.extra = {}
            if self.annotations:
                
                
                
                replacements = {
                    Ref(alias, annotation): annotation
                    for alias, annotation in self.annotations.items()
                }
                self.annotations = {
                    alias: aggregate.replace_expressions(replacements)
                    for alias, aggregate in aggregates.items()
                }
            else:
                self.annotations = aggregates
            self.set_annotation_mask(aggregates)

        empty_set_result = [
            expression.empty_result_set_value
            for expression in outer_query.annotation_select.values()
        ]
        elide_empty = not any(result is NotImplemented for result in empty_set_result)
        outer_query.clear_ordering(force=True)
        outer_query.clear_limits()
        outer_query.select_for_update = False
        outer_query.select_related = False
        compiler = outer_query.get_compiler(using, elide_empty=elide_empty)
        result = compiler.execute_sql(SINGLE)
        if result is None:
            result = empty_set_result
        else:
            converters = compiler.get_converters(outer_query.annotation_select.values())
            result = next(compiler.apply_converters((result,), converters))

        return dict(zip(outer_query.annotation_select, result))

    def get_count(self, using):
        
        obj = self.clone()
        return obj.get_aggregation(using, {"__count": Count("*")})["__count"]

    def has_filters(self):
        return self.where

    def exists(self, limit=True):
        q = self.clone()
        if not (q.distinct and q.is_sliced):
            if q.group_by is True:
                q.add_fields(
                    (f.attname for f in self.model._meta.concrete_fields), False
                )
                
                
                q.set_group_by(allow_aliases=False)
            q.clear_select_clause()
        if q.combined_queries and q.combinator == "union":
            q.combined_queries = tuple(
                combined_query.exists(limit=False)
                for combined_query in q.combined_queries
            )
        q.clear_ordering(force=True)
        if limit:
            q.set_limits(high=1)
        q.add_annotation(Value(1), "a")
        return q

    def has_results(self, using):
        q = self.exists(using)
        compiler = q.get_compiler(using=using)
        return compiler.has_results()

    def explain(self, using, format=None, **options):
        q = self.clone()
        for option_name in options:
            if (
                not EXPLAIN_OPTIONS_PATTERN.fullmatch(option_name)
                or "--" in option_name
            ):
                raise ValueError(f"Invalid option name: {option_name!r}.")
        q.explain_info = ExplainInfo(format, options)
        compiler = q.get_compiler(using=using)
        return "\n".join(compiler.explain_query())

    def combine(self, rhs, connector):
        
        if self.model != rhs.model:
            raise TypeError("Cannot combine queries on two different base models.")
        if self.is_sliced:
            raise TypeError("Cannot combine queries once a slice has been taken.")
        if self.distinct != rhs.distinct:
            raise TypeError("Cannot combine a unique query with a non-unique query.")
        if self.distinct_fields != rhs.distinct_fields:
            raise TypeError("Cannot combine queries with different distinct fields.")

        
        
        
        
        
        
        initial_alias = self.get_initial_alias()
        rhs.bump_prefix(self, exclude={initial_alias})

        
        change_map = {}
        conjunction = connector == AND

        
        
        
        
        
        
        
        
        
        
        
        
        reuse = set() if conjunction else set(self.alias_map)
        joinpromoter = JoinPromoter(connector, 2, False)
        joinpromoter.add_votes(
            j for j in self.alias_map if self.alias_map[j].join_type == INNER
        )
        rhs_votes = set()
        
        
        rhs_tables = list(rhs.alias_map)[1:]
        for alias in rhs_tables:
            join = rhs.alias_map[alias]
            
            
            join = join.relabeled_clone(change_map)
            new_alias = self.join(join, reuse=reuse)
            if join.join_type == INNER:
                rhs_votes.add(new_alias)
            
            
            
            reuse.discard(new_alias)
            if alias != new_alias:
                change_map[alias] = new_alias
            if not rhs.alias_refcount[alias]:
                
                
                
                
                self.unref_alias(new_alias)
        joinpromoter.add_votes(rhs_votes)
        joinpromoter.update_join_types(self)

        
        
        self.subq_aliases |= rhs.subq_aliases

        
        
        w = rhs.where.clone()
        w.relabel_aliases(change_map)
        self.where.add(w, connector)

        
        if rhs.select:
            self.set_select([col.relabeled_clone(change_map) for col in rhs.select])
        else:
            self.select = ()

        if connector == OR:
            
            
            
            if self.extra and rhs.extra:
                raise ValueError(
                    "When merging querysets using 'or', you cannot have "
                    "extra(select=...) on both sides."
                )
        self.extra.update(rhs.extra)
        extra_select_mask = set()
        if self.extra_select_mask is not None:
            extra_select_mask.update(self.extra_select_mask)
        if rhs.extra_select_mask is not None:
            extra_select_mask.update(rhs.extra_select_mask)
        if extra_select_mask:
            self.set_extra_mask(extra_select_mask)
        self.extra_tables += rhs.extra_tables

        
        
        self.order_by = rhs.order_by or self.order_by
        self.extra_order_by = rhs.extra_order_by or self.extra_order_by

    def _get_defer_select_mask(self, opts, mask, select_mask=None):
        if select_mask is None:
            select_mask = {}
        select_mask[opts.pk] = {}
        
        
        
        
        for field in opts.concrete_fields:
            field_mask = mask.pop(field.name, None)
            field_att_mask = mask.pop(field.attname, None)
            if field_mask is None and field_att_mask is None:
                select_mask.setdefault(field, {})
            elif field_mask:
                if not field.is_relation:
                    raise FieldError(next(iter(field_mask)))
                field_select_mask = select_mask.setdefault(field, {})
                related_model = field.remote_field.model._meta.concrete_model
                self._get_defer_select_mask(
                    related_model._meta, field_mask, field_select_mask
                )
        
        
        
        for field_name, field_mask in mask.items():
            if filtered_relation := self._filtered_relations.get(field_name):
                relation = opts.get_field(filtered_relation.relation_name)
                field_select_mask = select_mask.setdefault((field_name, relation), {})
                field = relation.field
            else:
                reverse_rel = opts.get_field(field_name)
                
                
                
                
                
                if not hasattr(reverse_rel, "field"):
                    continue
                field = reverse_rel.field
                field_select_mask = select_mask.setdefault(field, {})
            related_model = field.model._meta.concrete_model
            self._get_defer_select_mask(
                related_model._meta, field_mask, field_select_mask
            )
        return select_mask

    def _get_only_select_mask(self, opts, mask, select_mask=None):
        if select_mask is None:
            select_mask = {}
        select_mask[opts.pk] = {}
        
        for field_name, field_mask in mask.items():
            field = opts.get_field(field_name)
            
            
            if field in opts.related_objects:
                field_key = field.field
            else:
                field_key = field
            field_select_mask = select_mask.setdefault(field_key, {})
            if field_mask:
                if not field.is_relation:
                    raise FieldError(next(iter(field_mask)))
                related_model = field.remote_field.model._meta.concrete_model
                self._get_only_select_mask(
                    related_model._meta, field_mask, field_select_mask
                )
        return select_mask

    def get_select_mask(self):
        
        field_names, defer = self.deferred_loading
        if not field_names:
            return {}
        mask = {}
        for field_name in field_names:
            part_mask = mask
            for part in field_name.split(LOOKUP_SEP):
                part_mask = part_mask.setdefault(part, {})
        opts = self.get_meta()
        if defer:
            return self._get_defer_select_mask(opts, mask)
        return self._get_only_select_mask(opts, mask)

    def table_alias(self, table_name, create=False, filtered_relation=None):
        
        alias_list = self.table_map.get(table_name)
        if not create and alias_list:
            alias = alias_list[0]
            self.alias_refcount[alias] += 1
            return alias, False

        
        if alias_list:
            alias = "%s%d" % (self.alias_prefix, len(self.alias_map) + 1)
            alias_list.append(alias)
        else:
            
            alias = (
                filtered_relation.alias if filtered_relation is not None else table_name
            )
            self.table_map[table_name] = [alias]
        self.alias_refcount[alias] = 1
        return alias, True

    def ref_alias(self, alias):
        
        self.alias_refcount[alias] += 1

    def unref_alias(self, alias, amount=1):
        
        self.alias_refcount[alias] -= amount

    def promote_joins(self, aliases):
        
        aliases = list(aliases)
        while aliases:
            alias = aliases.pop(0)
            if self.alias_map[alias].join_type is None:
                
                
                
                continue
            
            assert self.alias_map[alias].join_type is not None
            parent_alias = self.alias_map[alias].parent_alias
            parent_louter = (
                parent_alias and self.alias_map[parent_alias].join_type == LOUTER
            )
            already_louter = self.alias_map[alias].join_type == LOUTER
            if (self.alias_map[alias].nullable or parent_louter) and not already_louter:
                self.alias_map[alias] = self.alias_map[alias].promote()
                
                
                aliases.extend(
                    join
                    for join in self.alias_map
                    if self.alias_map[join].parent_alias == alias
                    and join not in aliases
                )

    def demote_joins(self, aliases):
        
        aliases = list(aliases)
        while aliases:
            alias = aliases.pop(0)
            if self.alias_map[alias].join_type == LOUTER:
                self.alias_map[alias] = self.alias_map[alias].demote()
                parent_alias = self.alias_map[alias].parent_alias
                if self.alias_map[parent_alias].join_type == INNER:
                    aliases.append(parent_alias)

    def reset_refcounts(self, to_counts):
        
        for alias, cur_refcount in self.alias_refcount.copy().items():
            unref_amount = cur_refcount - to_counts.get(alias, 0)
            self.unref_alias(alias, unref_amount)

    def change_aliases(self, change_map):
        
        
        
        
        assert set(change_map).isdisjoint(change_map.values())

        
        
        self.where.relabel_aliases(change_map)
        if isinstance(self.group_by, tuple):
            self.group_by = tuple(
                [col.relabeled_clone(change_map) for col in self.group_by]
            )
        self.select = tuple([col.relabeled_clone(change_map) for col in self.select])
        self.annotations = self.annotations and {
            key: col.relabeled_clone(change_map)
            for key, col in self.annotations.items()
        }

        
        for old_alias, new_alias in change_map.items():
            if old_alias not in self.alias_map:
                continue
            alias_data = self.alias_map[old_alias].relabeled_clone(change_map)
            self.alias_map[new_alias] = alias_data
            self.alias_refcount[new_alias] = self.alias_refcount[old_alias]
            del self.alias_refcount[old_alias]
            del self.alias_map[old_alias]

            table_aliases = self.table_map[alias_data.table_name]
            for pos, alias in enumerate(table_aliases):
                if alias == old_alias:
                    table_aliases[pos] = new_alias
                    break
        self.external_aliases = {
            
            change_map.get(alias, alias): (aliased or alias in change_map)
            for alias, aliased in self.external_aliases.items()
        }

    def bump_prefix(self, other_query, exclude=None):
        

        def prefix_gen():
            
            alphabet = ascii_uppercase
            prefix = chr(ord(self.alias_prefix) + 1)
            yield prefix
            for n in count(1):
                seq = alphabet[alphabet.index(prefix) :] if prefix else alphabet
                for s in product(seq, repeat=n):
                    yield "".join(s)
                prefix = None

        if self.alias_prefix != other_query.alias_prefix:
            
            return

        
        
        
        
        local_recursion_limit = sys.getrecursionlimit() // 16
        for pos, prefix in enumerate(prefix_gen()):
            if prefix not in self.subq_aliases:
                self.alias_prefix = prefix
                break
            if pos > local_recursion_limit:
                raise RecursionError(
                    "Maximum recursion depth exceeded: too many subqueries."
                )
        self.subq_aliases = self.subq_aliases.union([self.alias_prefix])
        other_query.subq_aliases = other_query.subq_aliases.union(self.subq_aliases)
        if exclude is None:
            exclude = {}
        self.change_aliases(
            {
                alias: "%s%d" % (self.alias_prefix, pos)
                for pos, alias in enumerate(self.alias_map)
                if alias not in exclude
            }
        )

    def get_initial_alias(self):
        
        if self.alias_map:
            alias = self.base_table
            self.ref_alias(alias)
        elif self.model:
            alias = self.join(self.base_table_class(self.get_meta().db_table, None))
        else:
            alias = None
        return alias

    def count_active_tables(self):
        
        return len([1 for count in self.alias_refcount.values() if count])

    def join(self, join, reuse=None):
        
        reuse_aliases = [
            a
            for a, j in self.alias_map.items()
            if (reuse is None or a in reuse) and j == join
        ]
        if reuse_aliases:
            if join.table_alias in reuse_aliases:
                reuse_alias = join.table_alias
            else:
                
                
                reuse_alias = reuse_aliases[-1]
            self.ref_alias(reuse_alias)
            return reuse_alias

        
        alias, _ = self.table_alias(
            join.table_name, create=True, filtered_relation=join.filtered_relation
        )
        if join.join_type:
            if self.alias_map[join.parent_alias].join_type == LOUTER or join.nullable:
                join_type = LOUTER
            else:
                join_type = INNER
            join.join_type = join_type
        join.table_alias = alias
        self.alias_map[alias] = join
        if filtered_relation := join.filtered_relation:
            resolve_reuse = reuse
            if resolve_reuse is not None:
                resolve_reuse = set(reuse) | {alias}
            joins_len = len(self.alias_map)
            join.filtered_relation = filtered_relation.resolve_expression(
                self, reuse=resolve_reuse
            )
            
            
            if joins_len < len(self.alias_map):
                self.alias_map[alias] = self.alias_map.pop(alias)
        return alias

    def join_parent_model(self, opts, model, alias, seen):
        
        if model in seen:
            return seen[model]
        chain = opts.get_base_chain(model)
        if not chain:
            return alias
        curr_opts = opts
        for int_model in chain:
            if int_model in seen:
                curr_opts = int_model._meta
                alias = seen[int_model]
                continue
            
            
            
            
            if not curr_opts.parents[int_model]:
                curr_opts = int_model._meta
                continue
            link_field = curr_opts.get_ancestor_link(int_model)
            join_info = self.setup_joins([link_field.name], curr_opts, alias)
            curr_opts = int_model._meta
            alias = seen[int_model] = join_info.joins[-1]
        return alias or seen[None]

    def check_alias(self, alias):
        if FORBIDDEN_ALIAS_PATTERN.search(alias):
            raise ValueError(
                "Column aliases cannot contain whitespace characters, quotation marks, "
                "semicolons, or SQL comments."
            )

    def add_annotation(self, annotation, alias, select=True):
        
        self.check_alias(alias)
        annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None)
        if select:
            self.append_annotation_mask([alias])
        else:
            annotation_mask = (
                value
                for value in dict.fromkeys(self.annotation_select)
                if value != alias
            )
            self.set_annotation_mask(annotation_mask)
        self.annotations[alias] = annotation

    def resolve_expression(self, query, *args, **kwargs):
        clone = self.clone()
        
        clone.bump_prefix(query)
        clone.subquery = True
        clone.where.resolve_expression(query, *args, **kwargs)
        
        if clone.combinator:
            clone.combined_queries = tuple(
                [
                    combined_query.resolve_expression(query, *args, **kwargs)
                    for combined_query in clone.combined_queries
                ]
            )
        for key, value in clone.annotations.items():
            resolved = value.resolve_expression(query, *args, **kwargs)
            if hasattr(resolved, "external_aliases"):
                resolved.external_aliases.update(clone.external_aliases)
            clone.annotations[key] = resolved
        
        for alias, table in query.alias_map.items():
            clone.external_aliases[alias] = (
                isinstance(table, Join)
                and table.join_field.related_model._meta.db_table != alias
            ) or (
                isinstance(table, BaseTable) and table.table_name != table.table_alias
            )
        return clone

    def get_external_cols(self):
        exprs = chain(self.annotations.values(), self.where.children)
        return [
            col
            for col in self._gen_cols(exprs, include_external=True)
            if col.alias in self.external_aliases
        ]

    def get_group_by_cols(self, wrapper=None):
        
        
        
        
        external_cols = self.get_external_cols()
        if any(col.possibly_multivalued for col in external_cols):
            return [wrapper or self]
        return external_cols

    def as_sql(self, compiler, connection):
        
        
        if (
            self.subquery
            and not connection.features.ignores_unnecessary_order_by_in_subqueries
        ):
            self.clear_ordering(force=False)
            for query in self.combined_queries:
                query.clear_ordering(force=False)
        sql, params = self.get_compiler(connection=connection).as_sql()
        if self.subquery:
            sql = "(%s)" % sql
        return sql, params

    def resolve_lookup_value(self, value, can_reuse, allow_joins, summarize=False):
        if hasattr(value, "resolve_expression"):
            value = value.resolve_expression(
                self,
                reuse=can_reuse,
                allow_joins=allow_joins,
                summarize=summarize,
            )
        elif isinstance(value, (list, tuple)):
            
            
            values = (
                self.resolve_lookup_value(sub_value, can_reuse, allow_joins, summarize)
                for sub_value in value
            )
            type_ = type(value)
            if hasattr(type_, "_make"):  
                return type_(*values)
            return type_(values)
        return value

    def solve_lookup_type(self, lookup, summarize=False):
        
        lookup_splitted = lookup.split(LOOKUP_SEP)
        if self.annotations:
            annotation, expression_lookups = refs_expression(
                lookup_splitted, self.annotations
            )
            if annotation:
                expression = self.annotations[annotation]
                if summarize:
                    expression = Ref(annotation, expression)
                return expression_lookups, (), expression
        _, field, _, lookup_parts = self.names_to_path(lookup_splitted, self.get_meta())
        field_parts = lookup_splitted[0 : len(lookup_splitted) - len(lookup_parts)]
        if len(lookup_parts) > 1 and not field_parts:
            raise FieldError(
                'Invalid lookup "%s" for model %s".'
                % (lookup, self.get_meta().model.__name__)
            )
        return lookup_parts, field_parts, False

    def check_query_object_type(self, value, opts, field):
        
        if hasattr(value, "_meta"):
            if not check_rel_lookup_compatibility(value._meta.model, opts, field):
                raise ValueError(
                    'Cannot query "%s": Must be "%s" instance.'
                    % (value, opts.object_name)
                )

    def check_related_objects(self, field, value, opts):
        
        if field.is_relation:
            
            
            
            
            
            if (
                isinstance(value, Query)
                and not value.has_select_fields
                and not check_rel_lookup_compatibility(value.model, opts, field)
            ):
                raise ValueError(
                    'Cannot use QuerySet for "%s": Use a QuerySet for "%s".'
                    % (value.model._meta.object_name, opts.object_name)
                )
            elif hasattr(value, "_meta"):
                self.check_query_object_type(value, opts, field)
            elif hasattr(value, "__iter__"):
                for v in value:
                    self.check_query_object_type(v, opts, field)

    def check_filterable(self, expression):
        
        if hasattr(expression, "resolve_expression") and not getattr(
            expression, "filterable", True
        ):
            raise NotSupportedError(
                expression.__class__.__name__ + " is disallowed in the filter "
                "clause."
            )
        if hasattr(expression, "get_source_expressions"):
            for expr in expression.get_source_expressions():
                self.check_filterable(expr)

    def build_lookup(self, lookups, lhs, rhs):
        
        
        *transforms, lookup_name = lookups or ["exact"]
        for name in transforms:
            lhs = self.try_transform(lhs, name)
        
        
        lookup_class = lhs.get_lookup(lookup_name)
        if not lookup_class:
            
            
            lhs = self.try_transform(lhs, lookup_name)
            lookup_name = "exact"
            lookup_class = lhs.get_lookup(lookup_name)
            if not lookup_class:
                return

        lookup = lookup_class(lhs, rhs)
        
        
        if lookup.rhs is None and not lookup.can_use_none_as_rhs:
            if lookup_name not in ("exact", "iexact"):
                raise ValueError("Cannot use None as a query value")
            return lhs.get_lookup("isnull")(lhs, True)

        
        
        
        
        if (
            lookup_name == "exact"
            and lookup.rhs == ""
            and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
        ):
            return lhs.get_lookup("isnull")(lhs, True)

        return lookup

    def try_transform(self, lhs, name):
        
        transform_class = lhs.get_transform(name)
        if transform_class:
            return transform_class(lhs)
        else:
            output_field = lhs.output_field.__class__
            suggested_lookups = difflib.get_close_matches(
                name, lhs.output_field.get_lookups()
            )
            if suggested_lookups:
                suggestion = ", perhaps you meant %s?" % " or ".join(suggested_lookups)
            else:
                suggestion = "."
            raise FieldError(
                "Unsupported lookup '%s' for %s or join on the field not "
                "permitted%s" % (name, output_field.__name__, suggestion)
            )

    def build_filter(
        self,
        filter_expr,
        branch_negated=False,
        current_negated=False,
        can_reuse=None,
        allow_joins=True,
        split_subq=True,
        check_filterable=True,
        summarize=False,
        update_join_types=True,
    ):
        
        if isinstance(filter_expr, dict):
            raise FieldError("Cannot parse keyword query as dict")
        if isinstance(filter_expr, Q):
            return self._add_q(
                filter_expr,
                branch_negated=branch_negated,
                current_negated=current_negated,
                used_aliases=can_reuse,
                allow_joins=allow_joins,
                split_subq=split_subq,
                check_filterable=check_filterable,
                summarize=summarize,
                update_join_types=update_join_types,
            )
        if hasattr(filter_expr, "resolve_expression"):
            if not getattr(filter_expr, "conditional", False):
                raise TypeError("Cannot filter against a non-conditional expression.")
            condition = filter_expr.resolve_expression(
                self, allow_joins=allow_joins, reuse=can_reuse, summarize=summarize
            )
            if not isinstance(condition, Lookup):
                condition = self.build_lookup(["exact"], condition, True)
            return WhereNode([condition], connector=AND), []
        arg, value = filter_expr
        if not arg:
            raise FieldError("Cannot parse keyword query %r" % arg)
        lookups, parts, reffed_expression = self.solve_lookup_type(arg, summarize)

        if check_filterable:
            self.check_filterable(reffed_expression)

        if not allow_joins and len(parts) > 1:
            raise FieldError("Joined field references are not permitted in this query")

        pre_joins = self.alias_refcount.copy()
        value = self.resolve_lookup_value(value, can_reuse, allow_joins, summarize)
        used_joins = {
            k for k, v in self.alias_refcount.items() if v > pre_joins.get(k, 0)
        }

        if check_filterable:
            self.check_filterable(value)

        if reffed_expression:
            condition = self.build_lookup(lookups, reffed_expression, value)
            return WhereNode([condition], connector=AND), []

        opts = self.get_meta()
        alias = self.get_initial_alias()
        allow_many = not branch_negated or not split_subq

        try:
            join_info = self.setup_joins(
                parts,
                opts,
                alias,
                can_reuse=can_reuse,
                allow_many=allow_many,
            )

            
            if isinstance(value, Iterator):
                value = list(value)
            self.check_related_objects(join_info.final_field, value, join_info.opts)

            
            
            self._lookup_joins = join_info.joins
        except MultiJoin as e:
            return self.split_exclude(filter_expr, can_reuse, e.names_with_path)

        
        
        used_joins.update(join_info.joins)
        targets, alias, join_list = self.trim_joins(
            join_info.targets, join_info.joins, join_info.path
        )
        if can_reuse is not None:
            can_reuse.update(join_list)

        if join_info.final_field.is_relation:
            if len(targets) == 1:
                col = self._get_col(targets[0], join_info.final_field, alias)
            else:
                col = MultiColSource(
                    alias, targets, join_info.targets, join_info.final_field
                )
        else:
            col = self._get_col(targets[0], join_info.final_field, alias)

        condition = self.build_lookup(lookups, col, value)
        lookup_type = condition.lookup_name
        clause = WhereNode([condition], connector=AND)

        require_outer = (
            lookup_type == "isnull" and condition.rhs is True and not current_negated
        )
        if (
            current_negated
            and (lookup_type != "isnull" or condition.rhs is False)
            and condition.rhs is not None
        ):
            require_outer = True
            if lookup_type != "isnull":
                
                
                
                
                
                
                
                
                
                if (
                    self.is_nullable(targets[0])
                    or self.alias_map[join_list[-1]].join_type == LOUTER
                ):
                    lookup_class = targets[0].get_lookup("isnull")
                    col = self._get_col(targets[0], join_info.targets[0], alias)
                    clause.add(lookup_class(col, False), AND)
                
                
                if isinstance(value, Col) and self.is_nullable(value.target):
                    lookup_class = value.target.get_lookup("isnull")
                    clause.add(lookup_class(value, False), AND)
        return clause, used_joins if not require_outer else ()

    def add_filter(self, filter_lhs, filter_rhs):
        self.add_q(Q((filter_lhs, filter_rhs)))

    def add_q(self, q_object):
        
        
        
        
        
        
        
        existing_inner = {
            a for a in self.alias_map if self.alias_map[a].join_type == INNER
        }
        clause, _ = self._add_q(q_object, self.used_aliases)
        if clause:
            self.where.add(clause, AND)
        self.demote_joins(existing_inner)

    def build_where(self, filter_expr):
        return self.build_filter(filter_expr, allow_joins=False)[0]

    def clear_where(self):
        self.where = WhereNode()

    def _add_q(
        self,
        q_object,
        used_aliases,
        branch_negated=False,
        current_negated=False,
        allow_joins=True,
        split_subq=True,
        check_filterable=True,
        summarize=False,
        update_join_types=True,
    ):
        
        connector = q_object.connector
        current_negated ^= q_object.negated
        branch_negated = branch_negated or q_object.negated
        target_clause = WhereNode(connector=connector, negated=q_object.negated)
        joinpromoter = JoinPromoter(
            q_object.connector, len(q_object.children), current_negated
        )
        for child in q_object.children:
            child_clause, needed_inner = self.build_filter(
                child,
                can_reuse=used_aliases,
                branch_negated=branch_negated,
                current_negated=current_negated,
                allow_joins=allow_joins,
                split_subq=split_subq,
                check_filterable=check_filterable,
                summarize=summarize,
                update_join_types=update_join_types,
            )
            joinpromoter.add_votes(needed_inner)
            if child_clause:
                target_clause.add(child_clause, connector)
        if update_join_types:
            needed_inner = joinpromoter.update_join_types(self)
        else:
            needed_inner = []
        return target_clause, needed_inner

    def add_filtered_relation(self, filtered_relation, alias):
        filtered_relation.alias = alias
        relation_lookup_parts, relation_field_parts, _ = self.solve_lookup_type(
            filtered_relation.relation_name
        )
        if relation_lookup_parts:
            raise ValueError(
                "FilteredRelation's relation_name cannot contain lookups "
                "(got %r)." % filtered_relation.relation_name
            )
        for lookup in get_children_from_q(filtered_relation.condition):
            lookup_parts, lookup_field_parts, _ = self.solve_lookup_type(lookup)
            shift = 2 if not lookup_parts else 1
            lookup_field_path = lookup_field_parts[:-shift]
            for idx, lookup_field_part in enumerate(lookup_field_path):
                if len(relation_field_parts) > idx:
                    if relation_field_parts[idx] != lookup_field_part:
                        raise ValueError(
                            "FilteredRelation's condition doesn't support "
                            "relations outside the %r (got %r)."
                            % (filtered_relation.relation_name, lookup)
                        )
                else:
                    raise ValueError(
                        "FilteredRelation's condition doesn't support nested "
                        "relations deeper than the relation_name (got %r for "
                        "%r)." % (lookup, filtered_relation.relation_name)
                    )
        filtered_relation.condition = rename_prefix_from_q(
            filtered_relation.relation_name,
            alias,
            filtered_relation.condition,
        )
        self._filtered_relations[filtered_relation.alias] = filtered_relation

    def names_to_path(self, names, opts, allow_many=True, fail_on_missing=False):
        
        path, names_with_path = [], []
        for pos, name in enumerate(names):
            cur_names_with_path = (name, [])
            if name == "pk" and opts is not None:
                name = opts.pk.name

            field = None
            filtered_relation = None
            try:
                if opts is None:
                    raise FieldDoesNotExist
                field = opts.get_field(name)
            except FieldDoesNotExist:
                if name in self.annotation_select:
                    field = self.annotation_select[name].output_field
                elif name in self._filtered_relations and pos == 0:
                    filtered_relation = self._filtered_relations[name]
                    if LOOKUP_SEP in filtered_relation.relation_name:
                        parts = filtered_relation.relation_name.split(LOOKUP_SEP)
                        filtered_relation_path, field, _, _ = self.names_to_path(
                            parts,
                            opts,
                            allow_many,
                            fail_on_missing,
                        )
                        path.extend(filtered_relation_path[:-1])
                    else:
                        field = opts.get_field(filtered_relation.relation_name)
            if field is not None:
                
                
                
                if field.is_relation and not field.related_model:
                    raise FieldError(
                        "Field %r does not generate an automatic reverse "
                        "relation and therefore cannot be used for reverse "
                        "querying. If it is a GenericForeignKey, consider "
                        "adding a GenericRelation." % name
                    )
                try:
                    model = field.model._meta.concrete_model
                except AttributeError:
                    
                    
                    model = None
            else:
                
                
                pos -= 1
                if pos == -1 or fail_on_missing:
                    available = sorted(
                        [
                            *get_field_names_from_opts(opts),
                            *self.annotation_select,
                            *self._filtered_relations,
                        ]
                    )
                    raise FieldError(
                        "Cannot resolve keyword '%s' into field. "
                        "Choices are: %s" % (name, ", ".join(available))
                    )
                break
            
            
            
            if opts is not None and model is not opts.model:
                path_to_parent = opts.get_path_to_parent(model)
                if path_to_parent:
                    path.extend(path_to_parent)
                    cur_names_with_path[1].extend(path_to_parent)
                    opts = path_to_parent[-1].to_opts
            if hasattr(field, "path_infos"):
                if filtered_relation:
                    pathinfos = field.get_path_info(filtered_relation)
                else:
                    pathinfos = field.path_infos
                if not allow_many:
                    for inner_pos, p in enumerate(pathinfos):
                        if p.m2m:
                            cur_names_with_path[1].extend(pathinfos[0 : inner_pos + 1])
                            names_with_path.append(cur_names_with_path)
                            raise MultiJoin(pos + 1, names_with_path)
                last = pathinfos[-1]
                path.extend(pathinfos)
                final_field = last.join_field
                opts = last.to_opts
                targets = last.target_fields
                cur_names_with_path[1].extend(pathinfos)
                names_with_path.append(cur_names_with_path)
            else:
                
                final_field = field
                targets = (field,)
                if fail_on_missing and pos + 1 != len(names):
                    raise FieldError(
                        "Cannot resolve keyword %r into field. Join on '%s'"
                        " not permitted." % (names[pos + 1], name)
                    )
                break
        return path, final_field, targets, names[pos + 1 :]

    def setup_joins(
        self,
        names,
        opts,
        alias,
        can_reuse=None,
        allow_many=True,
    ):
        
        joins = [alias]
        
        
        
        

        def final_transformer(field, alias):
            if not self.alias_cols:
                alias = None
            return field.get_col(alias)

        
        
        last_field_exception = None
        for pivot in range(len(names), 0, -1):
            try:
                path, final_field, targets, rest = self.names_to_path(
                    names[:pivot],
                    opts,
                    allow_many,
                    fail_on_missing=True,
                )
            except FieldError as exc:
                if pivot == 1:
                    
                    
                    raise
                else:
                    last_field_exception = exc
            else:
                
                
                transforms = names[pivot:]
                break
        for name in transforms:

            def transform(field, alias, *, name, previous):
                try:
                    wrapped = previous(field, alias)
                    return self.try_transform(wrapped, name)
                except FieldError:
                    
                    if isinstance(final_field, Field) and last_field_exception:
                        raise last_field_exception
                    else:
                        raise

            final_transformer = functools.partial(
                transform, name=name, previous=final_transformer
            )
            final_transformer.has_transforms = True
        
        
        
        for join in path:
            if join.filtered_relation:
                filtered_relation = join.filtered_relation.clone()
                table_alias = filtered_relation.alias
            else:
                filtered_relation = None
                table_alias = None
            opts = join.to_opts
            if join.direct:
                nullable = self.is_nullable(join.join_field)
            else:
                nullable = True
            connection = self.join_class(
                opts.db_table,
                alias,
                table_alias,
                INNER,
                join.join_field,
                nullable,
                filtered_relation=filtered_relation,
            )
            reuse = can_reuse if join.m2m else None
            alias = self.join(connection, reuse=reuse)
            joins.append(alias)
        return JoinInfo(final_field, targets, opts, joins, path, final_transformer)

    def trim_joins(self, targets, joins, path):
        
        joins = joins[:]
        for pos, info in enumerate(reversed(path)):
            if len(joins) == 1 or not info.direct:
                break
            if info.filtered_relation:
                break
            join_targets = {t.column for t in info.join_field.foreign_related_fields}
            cur_targets = {t.column for t in targets}
            if not cur_targets.issubset(join_targets):
                break
            targets_dict = {
                r[1].column: r[0]
                for r in info.join_field.related_fields
                if r[1].column in cur_targets
            }
            targets = tuple(targets_dict[t.column] for t in targets)
            self.unref_alias(joins.pop())
        return targets, joins[-1], joins

    @classmethod
    def _gen_cols(cls, exprs, include_external=False, resolve_refs=True):
        for expr in exprs:
            if isinstance(expr, Col):
                yield expr
            elif include_external and callable(
                getattr(expr, "get_external_cols", None)
            ):
                yield from expr.get_external_cols()
            elif hasattr(expr, "get_source_expressions"):
                if not resolve_refs and isinstance(expr, Ref):
                    continue
                yield from cls._gen_cols(
                    expr.get_source_expressions(),
                    include_external=include_external,
                    resolve_refs=resolve_refs,
                )

    @classmethod
    def _gen_col_aliases(cls, exprs):
        yield from (expr.alias for expr in cls._gen_cols(exprs))

    def resolve_ref(self, name, allow_joins=True, reuse=None, summarize=False):
        annotation = self.annotations.get(name)
        if annotation is not None:
            if not allow_joins:
                for alias in self._gen_col_aliases([annotation]):
                    if isinstance(self.alias_map[alias], Join):
                        raise FieldError(
                            "Joined field references are not permitted in this query"
                        )
            if summarize:
                
                
                
                
                if name not in self.annotation_select:
                    raise FieldError(
                        "Cannot aggregate over the '%s' alias. Use annotate() "
                        "to promote it." % name
                    )
                return Ref(name, self.annotation_select[name])
            else:
                return annotation
        else:
            field_list = name.split(LOOKUP_SEP)
            annotation = self.annotations.get(field_list[0])
            if annotation is not None:
                for transform in field_list[1:]:
                    annotation = self.try_transform(annotation, transform)
                return annotation
            join_info = self.setup_joins(
                field_list, self.get_meta(), self.get_initial_alias(), can_reuse=reuse
            )
            targets, final_alias, join_list = self.trim_joins(
                join_info.targets, join_info.joins, join_info.path
            )
            if not allow_joins and len(join_list) > 1:
                raise FieldError(
                    "Joined field references are not permitted in this query"
                )
            if len(targets) > 1:
                raise FieldError(
                    "Referencing multicolumn fields with F() objects isn't supported"
                )
            
            
            transform = join_info.transform_function(targets[0], final_alias)
            if reuse is not None:
                reuse.update(join_list)
            return transform

    def split_exclude(self, filter_expr, can_reuse, names_with_path):
        
        
        query = self.__class__(self.model)
        query._filtered_relations = self._filtered_relations
        filter_lhs, filter_rhs = filter_expr
        if isinstance(filter_rhs, OuterRef):
            filter_rhs = OuterRef(filter_rhs)
        elif isinstance(filter_rhs, F):
            filter_rhs = OuterRef(filter_rhs.name)
        query.add_filter(filter_lhs, filter_rhs)
        query.clear_ordering(force=True)
        
        
        trimmed_prefix, contains_louter = query.trim_start(names_with_path)

        col = query.select[0]
        select_field = col.target
        alias = col.alias
        if alias in can_reuse:
            pk = select_field.model._meta.pk
            
            
            query.bump_prefix(self)
            lookup_class = select_field.get_lookup("exact")
            
            
            lookup = lookup_class(pk.get_col(query.select[0].alias), pk.get_col(alias))
            query.where.add(lookup, AND)
            query.external_aliases[alias] = True
        else:
            lookup_class = select_field.get_lookup("exact")
            lookup = lookup_class(col, ResolvedOuterRef(trimmed_prefix))
            query.where.add(lookup, AND)

        condition, needed_inner = self.build_filter(Exists(query))

        if contains_louter:
            or_null_condition, _ = self.build_filter(
                ("%s__isnull" % trimmed_prefix, True),
                current_negated=True,
                branch_negated=True,
                can_reuse=can_reuse,
            )
            condition.add(or_null_condition, OR)
            
            
            
            
            
        return condition, needed_inner

    def set_empty(self):
        self.where.add(NothingNode(), AND)
        for query in self.combined_queries:
            query.set_empty()

    def is_empty(self):
        return any(isinstance(c, NothingNode) for c in self.where.children)

    def set_limits(self, low=None, high=None):
        
        if high is not None:
            if self.high_mark is not None:
                self.high_mark = min(self.high_mark, self.low_mark + high)
            else:
                self.high_mark = self.low_mark + high
        if low is not None:
            if self.high_mark is not None:
                self.low_mark = min(self.high_mark, self.low_mark + low)
            else:
                self.low_mark = self.low_mark + low

        if self.low_mark == self.high_mark:
            self.set_empty()

    def clear_limits(self):
        
        self.low_mark, self.high_mark = 0, None

    @property
    def is_sliced(self):
        return self.low_mark != 0 or self.high_mark is not None

    def has_limit_one(self):
        return self.high_mark is not None and (self.high_mark - self.low_mark) == 1

    def can_filter(self):
        
        return not self.is_sliced

    def clear_select_clause(self):
        
        self.select = ()
        self.default_cols = False
        self.select_related = False
        self.set_extra_mask(())
        self.set_annotation_mask(())

    def clear_select_fields(self):
        
        self.select = ()
        self.values_select = ()

    def add_select_col(self, col, name):
        self.select += (col,)
        self.values_select += (name,)

    def set_select(self, cols):
        self.default_cols = False
        self.select = tuple(cols)

    def add_distinct_fields(self, *field_names):
        
        self.distinct_fields = field_names
        self.distinct = True

    def add_fields(self, field_names, allow_m2m=True):
        
        alias = self.get_initial_alias()
        opts = self.get_meta()

        try:
            cols = []
            for name in field_names:
                
                
                join_info = self.setup_joins(
                    name.split(LOOKUP_SEP), opts, alias, allow_many=allow_m2m
                )
                targets, final_alias, joins = self.trim_joins(
                    join_info.targets,
                    join_info.joins,
                    join_info.path,
                )
                for target in targets:
                    cols.append(join_info.transform_function(target, final_alias))
            if cols:
                self.set_select(cols)
        except MultiJoin:
            raise FieldError("Invalid field name: '%s'" % name)
        except FieldError:
            if LOOKUP_SEP in name:
                
                
                raise
            else:
                names = sorted(
                    [
                        *get_field_names_from_opts(opts),
                        *self.extra,
                        *self.annotation_select,
                        *self._filtered_relations,
                    ]
                )
                raise FieldError(
                    "Cannot resolve keyword %r into field. "
                    "Choices are: %s" % (name, ", ".join(names))
                )

    def add_ordering(self, *ordering):
        
        errors = []
        for item in ordering:
            if isinstance(item, str):
                if item == "?":
                    continue
                item = item.removeprefix("-")
                if item in self.annotations:
                    continue
                if self.extra and item in self.extra:
                    continue
                
                
                self.names_to_path(item.split(LOOKUP_SEP), self.model._meta)
            elif not hasattr(item, "resolve_expression"):
                errors.append(item)
            if getattr(item, "contains_aggregate", False):
                raise FieldError(
                    "Using an aggregate in order_by() without also including "
                    "it in annotate() is not allowed: %s" % item
                )
        if errors:
            raise FieldError("Invalid order_by arguments: %s" % errors)
        if ordering:
            self.order_by += ordering
        else:
            self.default_ordering = False

    def clear_ordering(self, force=False, clear_default=True):
        
        if not force and (
            self.is_sliced or self.distinct_fields or self.select_for_update
        ):
            return
        self.order_by = ()
        self.extra_order_by = ()
        if clear_default:
            self.default_ordering = False

    def set_group_by(self, allow_aliases=True):
        
        if allow_aliases and self.values_select:
            
            
            group_by_annotations = {}
            values_select = {}
            for alias, expr in zip(self.values_select, self.select):
                if isinstance(expr, Col):
                    values_select[alias] = expr
                else:
                    group_by_annotations[alias] = expr
            self.annotations = {**group_by_annotations, **self.annotations}
            self.append_annotation_mask(group_by_annotations)
            self.select = tuple(values_select.values())
            self.values_select = tuple(values_select)
        group_by = list(self.select)
        for alias, annotation in self.annotation_select.items():
            if not (group_by_cols := annotation.get_group_by_cols()):
                continue
            if allow_aliases and not annotation.contains_aggregate:
                group_by.append(Ref(alias, annotation))
            else:
                group_by.extend(group_by_cols)
        self.group_by = tuple(group_by)

    def add_select_related(self, fields):
        
        if isinstance(self.select_related, bool):
            field_dict = {}
        else:
            field_dict = self.select_related
        for field in fields:
            d = field_dict
            for part in field.split(LOOKUP_SEP):
                d = d.setdefault(part, {})
        self.select_related = field_dict

    def add_extra(self, select, select_params, where, params, tables, order_by):
        
        if select:
            
            
            
            
            select_pairs = {}
            if select_params:
                param_iter = iter(select_params)
            else:
                param_iter = iter([])
            for name, entry in select.items():
                self.check_alias(name)
                entry = str(entry)
                entry_params = []
                pos = entry.find("%s")
                while pos != -1:
                    if pos == 0 or entry[pos - 1] != "%":
                        entry_params.append(next(param_iter))
                    pos = entry.find("%s", pos + 2)
                select_pairs[name] = (entry, entry_params)
            self.extra.update(select_pairs)
        if where or params:
            self.where.add(ExtraWhere(where, params), AND)
        if tables:
            self.extra_tables += tuple(tables)
        if order_by:
            self.extra_order_by = order_by

    def clear_deferred_loading(self):
        
        self.deferred_loading = (frozenset(), True)

    def add_deferred_loading(self, field_names):
        
        
        
        
        
        existing, defer = self.deferred_loading
        if defer:
            
            self.deferred_loading = existing.union(field_names), True
        else:
            
            if new_existing := existing.difference(field_names):
                self.deferred_loading = new_existing, False
            else:
                self.clear_deferred_loading()
                if new_only := set(field_names).difference(existing):
                    self.deferred_loading = new_only, True

    def add_immediate_loading(self, field_names):
        
        existing, defer = self.deferred_loading
        field_names = set(field_names)
        if "pk" in field_names:
            field_names.remove("pk")
            field_names.add(self.get_meta().pk.name)

        if defer:
            
            
            self.deferred_loading = field_names.difference(existing), False
        else:
            
            self.deferred_loading = frozenset(field_names), False

    def set_annotation_mask(self, names):
        
        if names is None:
            self.annotation_select_mask = None
        else:
            self.annotation_select_mask = list(dict.fromkeys(names))
        self._annotation_select_cache = None

    def append_annotation_mask(self, names):
        if self.annotation_select_mask is not None:
            self.set_annotation_mask((*self.annotation_select_mask, *names))

    def set_extra_mask(self, names):
        
        if names is None:
            self.extra_select_mask = None
        else:
            self.extra_select_mask = set(names)
        self._extra_select_cache = None

    def set_values(self, fields):
        self.select_related = False
        self.clear_deferred_loading()
        self.clear_select_fields()
        self.has_select_fields = True

        if fields:
            field_names = []
            extra_names = []
            annotation_names = []
            if not self.extra and not self.annotations:
                
                
                field_names = list(fields)
            else:
                self.default_cols = False
                for f in fields:
                    if f in self.extra_select:
                        extra_names.append(f)
                    elif f in self.annotation_select:
                        annotation_names.append(f)
                    elif f in self.annotations:
                        raise FieldError(
                            f"Cannot select the '{f}' alias. Use annotate() to "
                            "promote it."
                        )
                    else:
                        
                        
                        
                        if self.annotation_select:
                            self.names_to_path(f.split(LOOKUP_SEP), self.model._meta)
                        field_names.append(f)
            self.set_extra_mask(extra_names)
            self.set_annotation_mask(annotation_names)
            selected = frozenset(field_names + extra_names + annotation_names)
        else:
            field_names = [f.attname for f in self.model._meta.concrete_fields]
            selected = frozenset(field_names)
        
        
        if self.group_by is True:
            self.add_fields(
                (f.attname for f in self.model._meta.concrete_fields), False
            )
            
            
            self.set_group_by(allow_aliases=False)
            self.clear_select_fields()
        elif self.group_by:
            
            
            group_by = []
            for expr in self.group_by:
                if isinstance(expr, Ref) and expr.refs not in selected:
                    expr = self.annotations[expr.refs]
                group_by.append(expr)
            self.group_by = tuple(group_by)

        self.values_select = tuple(field_names)
        self.add_fields(field_names, True)

    @property
    def annotation_select(self):
        
        if self._annotation_select_cache is not None:
            return self._annotation_select_cache
        elif not self.annotations:
            return {}
        elif self.annotation_select_mask is not None:
            self._annotation_select_cache = {
                k: self.annotations[k]
                for k in self.annotation_select_mask
                if k in self.annotations
            }
            return self._annotation_select_cache
        else:
            return self.annotations

    @property
    def extra_select(self):
        if self._extra_select_cache is not None:
            return self._extra_select_cache
        if not self.extra:
            return {}
        elif self.extra_select_mask is not None:
            self._extra_select_cache = {
                k: v for k, v in self.extra.items() if k in self.extra_select_mask
            }
            return self._extra_select_cache
        else:
            return self.extra

    def trim_start(self, names_with_path):
        
        all_paths = []
        for _, paths in names_with_path:
            all_paths.extend(paths)
        contains_louter = False
        
        
        
        lookup_tables = [
            t for t in self.alias_map if t in self._lookup_joins or t == self.base_table
        ]
        for trimmed_paths, path in enumerate(all_paths):
            if path.m2m:
                break
            if self.alias_map[lookup_tables[trimmed_paths + 1]].join_type == LOUTER:
                contains_louter = True
            alias = lookup_tables[trimmed_paths]
            self.unref_alias(alias)
        
        join_field = path.join_field.field
        
        paths_in_prefix = trimmed_paths
        trimmed_prefix = []
        for name, path in names_with_path:
            if paths_in_prefix - len(path) < 0:
                break
            trimmed_prefix.append(name)
            paths_in_prefix -= len(path)
        trimmed_prefix.append(join_field.foreign_related_fields[0].name)
        trimmed_prefix = LOOKUP_SEP.join(trimmed_prefix)
        
        
        
        
        
        
        first_join = self.alias_map[lookup_tables[trimmed_paths + 1]]
        if first_join.join_type != LOUTER and not first_join.filtered_relation:
            select_fields = [r[0] for r in join_field.related_fields]
            select_alias = lookup_tables[trimmed_paths + 1]
            self.unref_alias(lookup_tables[trimmed_paths])
            extra_restriction = join_field.get_extra_restriction(
                None, lookup_tables[trimmed_paths + 1]
            )
            if extra_restriction:
                self.where.add(extra_restriction, AND)
        else:
            
            
            
            select_fields = [r[1] for r in join_field.related_fields]
            select_alias = lookup_tables[trimmed_paths]
        
        
        
        for table in self.alias_map:
            if self.alias_refcount[table] > 0:
                self.alias_map[table] = self.base_table_class(
                    self.alias_map[table].table_name,
                    table,
                )
                break
        self.set_select([f.get_col(select_alias) for f in select_fields])
        return trimmed_prefix, contains_louter

    def is_nullable(self, field):
        
        
        
        
        
        
        return field.null or (
            field.empty_strings_allowed
            and connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls
        )


def get_order_dir(field, default="ASC"):
    
    dirn = ORDER_DIR[default]
    if field[0] == "-":
        return field[1:], dirn[1]
    return field, dirn[0]


class JoinPromoter:
    

    def __init__(self, connector, num_children, negated):
        self.connector = connector
        self.negated = negated
        if self.negated:
            if connector == AND:
                self.effective_connector = OR
            else:
                self.effective_connector = AND
        else:
            self.effective_connector = self.connector
        self.num_children = num_children
        
        
        self.votes = Counter()

    def __repr__(self):
        return (
            f"{self.__class__.__qualname__}(connector={self.connector!r}, "
            f"num_children={self.num_children!r}, negated={self.negated!r})"
        )

    def add_votes(self, votes):
        
        self.votes.update(votes)

    def update_join_types(self, query):
        
        to_promote = set()
        to_demote = set()
        
        
        for table, votes in self.votes.items():
            
            
            
            
            
            
            
            
            
            
            if self.effective_connector == OR and votes < self.num_children:
                to_promote.add(table)
            
            
            
            
            
            
            
            
            
            
            if self.effective_connector == AND or (
                self.effective_connector == OR and votes == self.num_children
            ):
                to_demote.add(table)
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
        query.promote_joins(to_promote)
        query.demote_joins(to_demote)
        return to_demote