Propose downstream triggered by comment loads config from dist-git
### What happened? What is the problem?

When propose downstream was triggered by a GitHub comment, the configuration was loaded from the dist-git repository instead of the GitHub repository (and since the defined branches for propose downstream differed, the job was triggered for incorrect branches).

Logs:
```
2023-04-16T06:50:30.400+0000 | Github issue#286 comment: '/packit propose-downstream' 'created' event.
2023-04-16T06:50:30.400+0000 | Target repo: scikit-build/scikit-build-core.
2023-04-16T06:50:35.232+0000 | Found a config file '.packit.yaml' on ref None of the 'rpms/python-scikit-build-core' repository.
2023-04-16T06:50:35.684+0000 |Package config before loading:
{
    "specfile_path": ".distro/scikit-build-core.spec",
    "files_to_sync": [
        {
            "src": ".distro/scikit-build-core.spec",
            "dest": "python-scikit-build-core.spec"
        },
        ".packit.yaml",
        {
            "src": ".distro/python-scikit-build-core.rpmlintrc",
            "dest": "python-scikit-build-core.rpmlintrc"
        }
    ],
    "upstream_package_name": "scikit_build_core",
    "downstream_package_name": "python-scikit-build-core",
    "update_release": false,
    "upstream_tag_template": "v{version}",
    "jobs": [
        {
            "job": "copr_build",
            "trigger": "pull_request",
            "owner": "@scikit-build",
            "project": "scikit-build-core",
            "update_release": true,
            "release_suffix": "{PACKIT_RPMSPEC_RELEASE}",
            "targets": [
                "fedora-development"
            ]
        },
        {
            "job": "copr_build",
            "trigger": "commit",
            "branch": "main",
            "owner": "@scikit-build",
            "project": "nightly",
            "targets": [
                "fedora-development-x86_64",
                "fedora-latest-x86_64",
                "fedora-development-aarch64",
                "fedora-latest-aarch64"
            ]
        },
        {
            "job": "copr_build",
            "trigger": "release",
            "owner": "@scikit-build",
            "project": "release",
            "targets": [
                "fedora-development-x86_64",
                "fedora-latest-x86_64",
                "fedora-development-aarch64",
                "fedora-latest-aarch64"
            ]
        },
        {
            "job": "propose_downstream",
            "trigger": "release",
            "dist_git_branches": [
                "fedora-development",
                "fedora-latest"
            ]
        },
     
```


### What did you expect to happen?

Configuration should be loaded from the upstream repository.

### Example URL(s)

https://github.com/scikit-build/scikit-build-core/issues/286
https://src.fedoraproject.org/rpms/python-scikit-build-core/pull-request/10#request_diff

### Steps to reproduce

```bash
1.
2.
3.
```


### What is the impacted category (job)?

Fedora release automation

### Workaround

- [ ] There is an existing workaround that can be used until this issue is fixed.

### Participation

- [ ] I am willing to submit a pull request for this issue. (Packit team is happy to help!)
diff --git a/packit_service/worker/jobs.py b/packit_service/worker/jobs.py
index 23982179..c8e1c5ee 100644
--- a/packit_service/worker/jobs.py
+++ b/packit_service/worker/jobs.py
@@ -330,14 +330,21 @@ class SteveJobs:
         Returns:
             Whether the Packit configuration is present in the repo.
         """
-        if isinstance(self.event, AbstractCommentEvent) and get_handlers_for_comment(
-            self.event.comment,
-            packit_comment_command_prefix=self.service_config.comment_command_prefix,
+        if isinstance(self.event, AbstractCommentEvent) and (
+            handlers := get_handlers_for_comment(
+                self.event.comment,
+                packit_comment_command_prefix=self.service_config.comment_command_prefix,
+            )
         ):
             # we require packit config file when event is triggered by /packit command
             # but not when it is triggered through an issue in the issues repository
             dist_git_package_config = None
-            if isinstance(self.event, AbstractIssueCommentEvent):
+            if (
+                isinstance(self.event, AbstractIssueCommentEvent)
+                # for propose-downstream we want to load the package config
+                # from upstream repo
+                and ProposeDownstreamHandler not in handlers
+            ):
                 if dist_git_package_config := self.search_distgit_config_in_issue():
                     (
                         self.event.dist_git_project_url,
diff --git a/tests/integration/test_issue_comment.py b/tests/integration/test_issue_comment.py
index c9884eaa..ed9c1559 100644
--- a/tests/integration/test_issue_comment.py
+++ b/tests/integration/test_issue_comment.py
@@ -13,14 +13,15 @@ from ogr.abstract import PRStatus
 from ogr.read_only import PullRequestReadOnly
 from ogr.services.github import GithubProject, GithubRelease
 from ogr.services.gitlab import GitlabProject, GitlabRelease
+from ogr.services.pagure import PagureProject
 
 from packit.api import PackitAPI
-from packit.config import JobConfigTriggerType
+from packit.config import JobConfigTriggerType, PackageConfig
 from packit.distgit import DistGit
 from packit.exceptions import PackitException
 from packit.local_project import LocalProject
 from packit.utils.koji_helper import KojiHelper
-from packit_service.config import ServiceConfig
+from packit_service.config import ServiceConfig, PackageConfigGetter
 from packit_service.constants import COMMENT_REACTION, TASK_ACCEPTED
 from packit_service.models import (
     IssueModel,
@@ -99,7 +100,19 @@ jobs:
         .with_args(author)
         .and_return(True)
     )
-    issue = flexmock(description="")
+    issue = flexmock(
+        description="Packit failed on creating pull-requests in dist-git "
+        "(https://src.fedoraproject.org/rpms/packit.git):"
+    )
+    flexmock(SteveJobs).should_receive("search_distgit_config_in_issue").never()
+    flexmock(PackageConfigGetter).should_call("get_package_config_from_repo").with_args(
+        base_project=None,
+        project=project_class,
+        reference="123456",
+        pr_id=None,
+        fail_when_missing=True,
+    ).and_return(PackageConfig)
+
     flexmock(project_class).should_receive("get_issue").and_return(issue)
     comment = flexmock()
     flexmock(issue).should_receive("get_comment").and_return(comment)
@@ -284,6 +297,10 @@ You can retrigger the update by adding a comment (`/packit propose-downstream`)
     project = (
         flexmock(GithubProject).should_receive("get_issue").and_return(issue).mock()
     )
+    flexmock(PackageConfigGetter).should_call("get_package_config_from_repo").with_args(
+        project=PagureProject, fail_when_missing=False
+    ).and_return(PackageConfig)
+
     project.should_receive("get_latest_release").and_return(flexmock(tag_name="123"))
     project.should_receive("get_sha_from_tag").and_return("abcdef")
     project.should_receive("has_write_access").and_return(True)
['tests/integration/test_issue_comment.py', 'packit_service/worker/jobs.py']