Add causal person fusion and MuVAP stage-3 baselines
This commit is contained in:
Xin Wang
6
third_party/EgoSpeak/.gitignore
vendored
Normal file
6
third_party/EgoSpeak/.gitignore
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
_legacy_/
|
||||
temp/
|
||||
visualization/
|
||||
backups/
|
||||
|
||||
*.egg-info
|
||||
1
third_party/EgoSpeak/.upstream-revision
vendored
Normal file
1
third_party/EgoSpeak/.upstream-revision
vendored
Normal file
@@ -0,0 +1 @@
|
||||
b5bee35e796d8f2d6aa7da2bf1b07836270f8bfb
|
||||
35
third_party/EgoSpeak/LSTR/.gitignore
vendored
Normal file
35
third_party/EgoSpeak/LSTR/.gitignore
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
*.zip
|
||||
backup/
|
||||
# *.out
|
||||
*-log
|
||||
temp/
|
||||
|
||||
scripts/
|
||||
# Miscellaneous
|
||||
*.DS_Store
|
||||
*.swp
|
||||
*.pyc
|
||||
*.egg-info
|
||||
|
||||
# PyTorch models
|
||||
*.pth
|
||||
checkpoints/
|
||||
|
||||
# Dataset soft links
|
||||
data/THUMOS
|
||||
data/TVSeries
|
||||
data/HACS
|
||||
data/EasyCom
|
||||
data/EGO4D
|
||||
data/EGO4D
|
||||
|
||||
# logs
|
||||
log_*
|
||||
wandb/
|
||||
*.egg-info
|
||||
slurm-*.out
|
||||
|
||||
*.mp4
|
||||
|
||||
wandb/
|
||||
git_diff/
|
||||
3
third_party/EgoSpeak/LSTR/.gitmodules
vendored
Normal file
3
third_party/EgoSpeak/LSTR/.gitmodules
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[submodule "external/rulstm"]
|
||||
path = external/rulstm
|
||||
url = git@github.com:fpv-iplab/rulstm.git
|
||||
175
third_party/EgoSpeak/LSTR/LICENSE
vendored
Normal file
175
third_party/EgoSpeak/LSTR/LICENSE
vendored
Normal file
@@ -0,0 +1,175 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
9
third_party/EgoSpeak/LSTR/NOTICE
vendored
Normal file
9
third_party/EgoSpeak/LSTR/NOTICE
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
|
||||
|
||||
This codebase is based on the following repository:
|
||||
https://github.com/amazon-science/long-short-term-transformer
|
||||
|
||||
The codebase is modified from the original repository to fit the needs of the LSTR model.
|
||||
|
||||
|
||||
44
third_party/EgoSpeak/LSTR/configs/EasyCom/LSTR/lstr_aa_easycom.yaml
vendored
Normal file
44
third_party/EgoSpeak/LSTR/configs/EasyCom/LSTR/lstr_aa_easycom.yaml
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
DATA:
|
||||
DATA_NAME: 'EasyCom'
|
||||
INPUT:
|
||||
MODALITY: 'visual+motion' # visual+motion+object
|
||||
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
|
||||
MOTION_FEATURE: 'audio_wav2vec2' # audio
|
||||
OBJECT_FEATURE: 'audio_wav2vec2'
|
||||
TARGET_PERFRAME: 'target_perframe' # 3 classes
|
||||
MODEL:
|
||||
MODEL_NAME: 'LSTR'
|
||||
FEATURE_HEAD:
|
||||
LINEAR_ENABLED: True
|
||||
LINEAR_OUT_FEATURES: 1024
|
||||
LSTR:
|
||||
NUM_HEADS: 16
|
||||
DIM_FEEDFORWARD: 1024
|
||||
DROPOUT: 0.2
|
||||
ACTIVATION: 'relu'
|
||||
LONG_MEMORY_SECONDS: 512
|
||||
LONG_MEMORY_SAMPLE_RATE: 4
|
||||
LONG_MEMORY_USE_PE: True
|
||||
WORK_MEMORY_SECONDS: 8
|
||||
WORK_MEMORY_SAMPLE_RATE: 1
|
||||
ANTICIPATION_SECONDS: 2
|
||||
ANTICIPATION_SAMPLE_RATE: 1
|
||||
ENC_MODULE: [
|
||||
[16, 1, True], [32, 2, True]
|
||||
]
|
||||
DEC_MODULE: [-1, 2, True]
|
||||
CRITERIONS: [['MCE', {}]]
|
||||
DATA_LOADER:
|
||||
BATCH_SIZE: 16
|
||||
NUM_WORKERS: 8
|
||||
PIN_MEMORY: True
|
||||
SOLVER:
|
||||
NUM_EPOCHS: 50
|
||||
OPTIMIZER: 'adam'
|
||||
BASE_LR: 7e-05
|
||||
WEIGHT_DECAY: 5e-05
|
||||
SCHEDULER:
|
||||
SCHEDULER_NAME: 'warmup_cosine'
|
||||
WARMUP_FACTOR: 0.3
|
||||
WARMUP_EPOCHS: 10.0
|
||||
WARMUP_METHOD: 'linear'
|
||||
45
third_party/EgoSpeak/LSTR/configs/EasyCom/LSTR/lstr_aa_easycom_from_ytturn.yaml
vendored
Normal file
45
third_party/EgoSpeak/LSTR/configs/EasyCom/LSTR/lstr_aa_easycom_from_ytturn.yaml
vendored
Normal file
@@ -0,0 +1,45 @@
|
||||
DATA:
|
||||
DATA_NAME: 'EasyCom'
|
||||
INPUT:
|
||||
MODALITY: 'visual+motion' # visual+motion+object
|
||||
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
|
||||
MOTION_FEATURE: 'audio_wav2vec2' # audio
|
||||
OBJECT_FEATURE: 'audio_wav2vec2'
|
||||
TARGET_PERFRAME: 'target_perframe' # 3 classes
|
||||
MODEL:
|
||||
CHECKPOINT: 'checkpoints/lstr_ytconv_pretrained_AV.pth'
|
||||
MODEL_NAME: 'LSTR'
|
||||
FEATURE_HEAD:
|
||||
LINEAR_ENABLED: True
|
||||
LINEAR_OUT_FEATURES: 1024
|
||||
LSTR:
|
||||
NUM_HEADS: 16
|
||||
DIM_FEEDFORWARD: 1024
|
||||
DROPOUT: 0.2
|
||||
ACTIVATION: 'relu'
|
||||
LONG_MEMORY_SECONDS: 512
|
||||
LONG_MEMORY_SAMPLE_RATE: 4
|
||||
LONG_MEMORY_USE_PE: True
|
||||
WORK_MEMORY_SECONDS: 8
|
||||
WORK_MEMORY_SAMPLE_RATE: 1
|
||||
ANTICIPATION_SECONDS: 2
|
||||
ANTICIPATION_SAMPLE_RATE: 1
|
||||
ENC_MODULE: [
|
||||
[16, 1, True], [32, 2, True]
|
||||
]
|
||||
DEC_MODULE: [-1, 2, True]
|
||||
CRITERIONS: [['MCE', {}]]
|
||||
DATA_LOADER:
|
||||
BATCH_SIZE: 16
|
||||
NUM_WORKERS: 8
|
||||
PIN_MEMORY: True
|
||||
SOLVER:
|
||||
NUM_EPOCHS: 50
|
||||
OPTIMIZER: 'adam'
|
||||
BASE_LR: 7e-05
|
||||
WEIGHT_DECAY: 5e-05
|
||||
SCHEDULER:
|
||||
SCHEDULER_NAME: 'warmup_cosine'
|
||||
WARMUP_FACTOR: 0.3
|
||||
WARMUP_EPOCHS: 10.0
|
||||
WARMUP_METHOD: 'linear'
|
||||
46
third_party/EgoSpeak/LSTR/configs/YTConv/LSTR/lstr_aa_ytconv.yaml
vendored
Normal file
46
third_party/EgoSpeak/LSTR/configs/YTConv/LSTR/lstr_aa_ytconv.yaml
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
DATA:
|
||||
DATA_NAME: 'YTConv'
|
||||
DATA_INFO: 'data/YTConv_info.json'
|
||||
INPUT:
|
||||
MODALITY: 'visual+motion' # visual+motion+object
|
||||
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
|
||||
MOTION_FEATURE: 'audio_wav2vec2' # audio
|
||||
OBJECT_FEATURE: 'audio_wav2vec2'
|
||||
TARGET_PERFRAME: 'target_perframe' # 3 classes
|
||||
MODEL:
|
||||
MODEL_NAME: 'LSTR'
|
||||
FEATURE_HEAD:
|
||||
LINEAR_ENABLED: True
|
||||
LINEAR_OUT_FEATURES: 1024
|
||||
LSTR:
|
||||
NUM_HEADS: 16
|
||||
DIM_FEEDFORWARD: 1024
|
||||
DROPOUT: 0.2
|
||||
ACTIVATION: 'relu'
|
||||
LONG_MEMORY_SECONDS: 512
|
||||
LONG_MEMORY_SAMPLE_RATE: 4
|
||||
LONG_MEMORY_USE_PE: True
|
||||
WORK_MEMORY_SECONDS: 8
|
||||
WORK_MEMORY_SAMPLE_RATE: 1
|
||||
ANTICIPATION_SECONDS: 2
|
||||
ANTICIPATION_SAMPLE_RATE: 1
|
||||
ENC_MODULE: [
|
||||
[16, 1, True], [32, 2, True]
|
||||
]
|
||||
DEC_MODULE: [-1, 2, True]
|
||||
CRITERIONS: [['MCE', {}]]
|
||||
DATA_LOADER:
|
||||
BATCH_SIZE: 16
|
||||
NUM_WORKERS: 8
|
||||
PIN_MEMORY: True
|
||||
SOLVER:
|
||||
SAVE_EVERY_EPOCHS: 5
|
||||
NUM_EPOCHS: 50
|
||||
OPTIMIZER: 'adam'
|
||||
BASE_LR: 7e-05
|
||||
WEIGHT_DECAY: 5e-05
|
||||
SCHEDULER:
|
||||
SCHEDULER_NAME: 'warmup_cosine'
|
||||
WARMUP_FACTOR: 0.3
|
||||
WARMUP_EPOCHS: 10.0
|
||||
WARMUP_METHOD: 'linear'
|
||||
45
third_party/EgoSpeak/LSTR/configs/ego4d/LSTR/lstr_aa_ego4dshuffle.yaml
vendored
Normal file
45
third_party/EgoSpeak/LSTR/configs/ego4d/LSTR/lstr_aa_ego4dshuffle.yaml
vendored
Normal file
@@ -0,0 +1,45 @@
|
||||
DATA:
|
||||
DATA_NAME: 'EGO4DSHUFFLE'
|
||||
# IGNORE_INDEX: 0 # calculate mAP w/o target not speaking (background)
|
||||
INPUT:
|
||||
MODALITY: 'visual+motion' # audio+visual
|
||||
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
|
||||
MOTION_FEATURE: 'audio_wav2vec2' # audio
|
||||
OBJECT_FEATURE: 'rgb_kinetics_resnet50'
|
||||
TARGET_PERFRAME: 'target_perframe' # 3 classes
|
||||
MODEL:
|
||||
MODEL_NAME: 'LSTR'
|
||||
FEATURE_HEAD:
|
||||
LINEAR_ENABLED: True
|
||||
LINEAR_OUT_FEATURES: 1024
|
||||
LSTR:
|
||||
NUM_HEADS: 16
|
||||
DIM_FEEDFORWARD: 1024
|
||||
DROPOUT: 0.2
|
||||
ACTIVATION: 'relu'
|
||||
LONG_MEMORY_SECONDS: 512
|
||||
LONG_MEMORY_SAMPLE_RATE: 4
|
||||
LONG_MEMORY_USE_PE: True
|
||||
WORK_MEMORY_SECONDS: 8
|
||||
WORK_MEMORY_SAMPLE_RATE: 1
|
||||
ANTICIPATION_SECONDS: 2
|
||||
ANTICIPATION_SAMPLE_RATE: 1
|
||||
ENC_MODULE: [
|
||||
[16, 1, True], [32, 2, True]
|
||||
]
|
||||
DEC_MODULE: [-1, 2, True]
|
||||
CRITERIONS: [['MCE', {}]]
|
||||
DATA_LOADER:
|
||||
BATCH_SIZE: 16
|
||||
NUM_WORKERS: 8
|
||||
PIN_MEMORY: True
|
||||
SOLVER:
|
||||
NUM_EPOCHS: 50
|
||||
OPTIMIZER: 'adam'
|
||||
BASE_LR: 7e-05
|
||||
WEIGHT_DECAY: 5e-05
|
||||
SCHEDULER:
|
||||
SCHEDULER_NAME: 'warmup_cosine'
|
||||
WARMUP_FACTOR: 0.3
|
||||
WARMUP_EPOCHS: 10.0
|
||||
WARMUP_METHOD: 'linear'
|
||||
46
third_party/EgoSpeak/LSTR/configs/ego4d/LSTR/lstr_aa_ego4dshuffle_from_ytconv.yaml
vendored
Normal file
46
third_party/EgoSpeak/LSTR/configs/ego4d/LSTR/lstr_aa_ego4dshuffle_from_ytconv.yaml
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
DATA:
|
||||
DATA_NAME: 'EGO4DSHUFFLE'
|
||||
# IGNORE_INDEX: 0 # calculate mAP w/o target not speaking (background)
|
||||
INPUT:
|
||||
MODALITY: 'visual+motion' # audio+visual
|
||||
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
|
||||
MOTION_FEATURE: 'audio_wav2vec2' # audio
|
||||
OBJECT_FEATURE: 'rgb_kinetics_resnet50'
|
||||
TARGET_PERFRAME: 'target_perframe' # 3 classes
|
||||
MODEL:
|
||||
CHECKPOINT: 'checkpoints/lstr_ytconv_pretrained_AV.pth'
|
||||
MODEL_NAME: 'LSTR'
|
||||
FEATURE_HEAD:
|
||||
LINEAR_ENABLED: True
|
||||
LINEAR_OUT_FEATURES: 1024
|
||||
LSTR:
|
||||
NUM_HEADS: 16
|
||||
DIM_FEEDFORWARD: 1024
|
||||
DROPOUT: 0.2
|
||||
ACTIVATION: 'relu'
|
||||
LONG_MEMORY_SECONDS: 512
|
||||
LONG_MEMORY_SAMPLE_RATE: 4
|
||||
LONG_MEMORY_USE_PE: True
|
||||
WORK_MEMORY_SECONDS: 8
|
||||
WORK_MEMORY_SAMPLE_RATE: 1
|
||||
ANTICIPATION_SECONDS: 2
|
||||
ANTICIPATION_SAMPLE_RATE: 1
|
||||
ENC_MODULE: [
|
||||
[16, 1, True], [32, 2, True]
|
||||
]
|
||||
DEC_MODULE: [-1, 2, True]
|
||||
CRITERIONS: [['MCE', {}]]
|
||||
DATA_LOADER:
|
||||
BATCH_SIZE: 16
|
||||
NUM_WORKERS: 8
|
||||
PIN_MEMORY: True
|
||||
SOLVER:
|
||||
NUM_EPOCHS: 50
|
||||
OPTIMIZER: 'adam'
|
||||
BASE_LR: 7e-05
|
||||
WEIGHT_DECAY: 5e-05
|
||||
SCHEDULER:
|
||||
SCHEDULER_NAME: 'warmup_cosine'
|
||||
WARMUP_FACTOR: 0.3
|
||||
WARMUP_EPOCHS: 10.0
|
||||
WARMUP_METHOD: 'linear'
|
||||
431
third_party/EgoSpeak/LSTR/data/YTConv_info.json
vendored
Normal file
431
third_party/EgoSpeak/LSTR/data/YTConv_info.json
vendored
Normal file
@@ -0,0 +1,431 @@
|
||||
{
|
||||
"YTConv": {
|
||||
"data_root": "data/YTConv",
|
||||
"class_names": [
|
||||
"background",
|
||||
"camera wearer start to speak"
|
||||
],
|
||||
"num_classes": 2,
|
||||
"ignore_index": -100,
|
||||
"metrics": "AP",
|
||||
"fps": 5,
|
||||
"train_session_set": [
|
||||
"m5Aoywf51L4",
|
||||
"p99cIB7nHjw",
|
||||
"zGgMzBfPfaI",
|
||||
"O0eCtKU3FxY",
|
||||
"goZQ6iOIFYo",
|
||||
"9Tb-lSyhgoE",
|
||||
"GBxQtGUjAMA",
|
||||
"VXgt7keOC8I",
|
||||
"hZ2tin2J0P4",
|
||||
"Ur-P18sm4tI",
|
||||
"WfYmsNBQwg8",
|
||||
"AvMNaDDfrN0",
|
||||
"fW7pCN3wwdo",
|
||||
"xG6HBdTtS4s",
|
||||
"HcDlqUeDD9Q",
|
||||
"1DE3mKncHUA",
|
||||
"oEMX_quSMZY",
|
||||
"jmS3nWbGV5Y",
|
||||
"qUJRjPRH-u4",
|
||||
"aHsUgWfUcss",
|
||||
"ZfdQOmJkaK0",
|
||||
"4f43nYrfTnI",
|
||||
"qE1FnzkNjrE",
|
||||
"4NsmKhs9flk",
|
||||
"3wDSJ9ABzmo",
|
||||
"sY6_8QwRZsw",
|
||||
"hByfAzEoeKA",
|
||||
"vDEsV0AhcjM",
|
||||
"XGNhX3Nu1RE",
|
||||
"MkVT8K0V57A",
|
||||
"yR8y-wN8Clk",
|
||||
"P1dTOlO2sl8",
|
||||
"6UGC_wxTeB0",
|
||||
"d4Xud2Gqoxk",
|
||||
"IJ1aqePculQ",
|
||||
"o8Oxf5QBitQ",
|
||||
"XVci_V4qxmc",
|
||||
"ofDzhOS1Pv0",
|
||||
"mygTXBuMRF0",
|
||||
"iq6q82Jo8ZQ",
|
||||
"Kzc_ekBJ4QE",
|
||||
"WEeJ9B1i8HI",
|
||||
"rciqpxdCc9Y",
|
||||
"Rh7d2D7JVLI",
|
||||
"Qkx4hSlc8jk",
|
||||
"ixi7sVLFrWM",
|
||||
"n96-scSyoOQ",
|
||||
"YZ5xYioIQdA",
|
||||
"X76QxuWm1oA",
|
||||
"2I7d_LfK75Y",
|
||||
"AeUWBaYFMyo",
|
||||
"lsqXFgymLlI",
|
||||
"Dx_0jYMWN5o",
|
||||
"0raJ1-1BJ7s",
|
||||
"EGX8am5hWh0",
|
||||
"KjtZ_CLY1hQ",
|
||||
"R4FQI2BI3hw",
|
||||
"fAoqqtsNjbQ",
|
||||
"l6RwyA6jUdM",
|
||||
"GJW6dlagRnU",
|
||||
"j_FGvmbp6aM",
|
||||
"V2O15h1VrH8",
|
||||
"P3c70VbDP3E",
|
||||
"epAwpBdUiYU",
|
||||
"6dpv_JeyL8w",
|
||||
"dUKYxsvnuBM",
|
||||
"L2_zn7h8sOQ",
|
||||
"m09qdCkAhmY",
|
||||
"7BB71tIae7A",
|
||||
"6go4oViEkdc",
|
||||
"MYimxshUK1c",
|
||||
"v4eZXWcIRQU",
|
||||
"vGI_w1Ml6S0",
|
||||
"BeFndnkT00c",
|
||||
"9JHRbL1jmq4",
|
||||
"0rBaR5dd-Fs",
|
||||
"EvhkIS_-K-w",
|
||||
"pYVbLx-h5HI",
|
||||
"fVciOXYDlqg",
|
||||
"zWFH4N-CmDE",
|
||||
"Aw4cauzfX5E",
|
||||
"lxVpcQHtcWs",
|
||||
"32dkF-aXgug",
|
||||
"_USorgWOsUk",
|
||||
"1yVMU93nmv8",
|
||||
"aYkF4Ybk-l8",
|
||||
"Vwt18p0BqzM",
|
||||
"oOQM5Pvjut4",
|
||||
"_WWSMNtNYx0",
|
||||
"zzlhwg0IJ5Y",
|
||||
"lJhIffkUhr8",
|
||||
"PbS9nSH3mqM",
|
||||
"8EGVgiFpwhg",
|
||||
"5LhXpv5IFO4",
|
||||
"ibD4APVzWgg",
|
||||
"2nhcSQmopO4",
|
||||
"gri79OfCRvo",
|
||||
"lOwQN3QcRzE",
|
||||
"wQyluRNtqFo",
|
||||
"IZyx8zeiKB0",
|
||||
"mZyNfKyzDHU",
|
||||
"A9eGJWiuIKs",
|
||||
"d37DZogot1w",
|
||||
"7ZUSVabB6N4",
|
||||
"m62GQXq-358",
|
||||
"ArwPVsCZo2c",
|
||||
"QK-BFkpO8s4",
|
||||
"PgfX9bvmEhs",
|
||||
"l1jTUhwWJYA",
|
||||
"S1PGHO4evaI",
|
||||
"M6uPJa5B8Ns",
|
||||
"XXzz19J-1w8",
|
||||
"U4uV1xmUaB0",
|
||||
"skQ0OBpkKqY",
|
||||
"mfW57TDpMtc",
|
||||
"NAwKA75cDwE",
|
||||
"eIEBI2_Y14Q",
|
||||
"eWQGYpSoiYU",
|
||||
"3DPIUWhYtn8",
|
||||
"_elvaRJrhE0",
|
||||
"PAA8IFhWJzM",
|
||||
"1XDKqhq6bb0",
|
||||
"ngi9gy3DzRY",
|
||||
"Y0yONfuimeE",
|
||||
"g9_cAgHa_4Q",
|
||||
"BIG7jEw-j4w",
|
||||
"TFdom0lkd-0",
|
||||
"2zV2N8_DwWg",
|
||||
"kTz7XB_jpB4",
|
||||
"GuoglIuhNjY",
|
||||
"L98U3mO38gQ",
|
||||
"UfWDoRAzpC8",
|
||||
"W6TakaXCIIQ",
|
||||
"i6j1I6pha-M",
|
||||
"issHFC8Gbio",
|
||||
"4EzGoRkd6SI",
|
||||
"QoS0GgF4sI4",
|
||||
"A8hJg3D_TGY",
|
||||
"A3N10suCRCw",
|
||||
"6962tnWRsNY",
|
||||
"lS4p8osXSJs",
|
||||
"wbZ7ZhcZd6o",
|
||||
"fsIvQvIlKwc",
|
||||
"4jQSOSi2DA8",
|
||||
"Vpt6tZBvXQU",
|
||||
"kxOG3kpLmYw",
|
||||
"Mzjlz0nq9AA",
|
||||
"80cuXt2VSko",
|
||||
"VvswhogSiY0",
|
||||
"5Q6ky5VrOIA",
|
||||
"I_XtcUpNRF8",
|
||||
"QmtmTGycK9A",
|
||||
"qp3x8iy7vkA",
|
||||
"xGU8QDtPrQI",
|
||||
"uWWUfqth_BA",
|
||||
"WbM7w8r4VGQ",
|
||||
"bt6Ed1qjulw",
|
||||
"7vymT27scoY",
|
||||
"b9AdV6oVDVg",
|
||||
"nf2zqYUAOwg",
|
||||
"A-LNJc-1Jc0",
|
||||
"yCI-hpPiDVc",
|
||||
"Uu5DME4FqIM",
|
||||
"symjuH9iOTc",
|
||||
"mPf2MTnTBc0",
|
||||
"NDTlQ1KGvH8",
|
||||
"g9vuDzFQSRU",
|
||||
"B6iqnvleve0",
|
||||
"9bbszAD7JL0",
|
||||
"RIyMAuz7Ckg",
|
||||
"mlrQHtBRMhc",
|
||||
"3gaidlW7vig",
|
||||
"DGdV97q9ndk",
|
||||
"qthjlg0lXFg",
|
||||
"gWng5YLPOow",
|
||||
"iaYG-KL9XRM",
|
||||
"385414AVZcA",
|
||||
"7xhBnWGpI7c",
|
||||
"Nh0cmSQUs-E",
|
||||
"Rvtj0QHPulA",
|
||||
"rAtLksYQukw",
|
||||
"S9rB__zbbe0",
|
||||
"SNpMVsOPfHM",
|
||||
"zhF43F4kO2w",
|
||||
"HOUkcInyxN0",
|
||||
"x8ymkfr2Z1o",
|
||||
"Ix6Khtq6O-k",
|
||||
"VHs_JyPhmtE",
|
||||
"zJX3byjYSMc",
|
||||
"zdqX09zZNf8",
|
||||
"uPwlmwNQq8Y",
|
||||
"e4xNTXrk6wE",
|
||||
"P2PNUWN4GMs",
|
||||
"_WVHCYjHBVY",
|
||||
"5Jtv1GpPvHM",
|
||||
"AIZYucKOzhw",
|
||||
"vbsR_VDZXh0",
|
||||
"9Ew0lDHB0kM",
|
||||
"ejpkPVZHuMM",
|
||||
"47HlvCfubE4",
|
||||
"HzPkQSIG2m0",
|
||||
"e7mTRnNnOXg",
|
||||
"eZHx_biEECg",
|
||||
"FvP6WFKNP4Y",
|
||||
"g_TJ5XFEVfY",
|
||||
"8Utg2VEWsJg",
|
||||
"VMdBjL-tpoo",
|
||||
"zihP2Eqeqxk",
|
||||
"ILVqgOlntLM",
|
||||
"Z1BWQI8KPCg",
|
||||
"j8TRVp-7l3Q",
|
||||
"G5gJBZ-3OlY",
|
||||
"o1JUlGSirSE",
|
||||
"OsPdhUQ0snE",
|
||||
"-5uIaW2gKWQ",
|
||||
"03kiFNh-9ME",
|
||||
"w5W6mgDaJJA",
|
||||
"sYsgd_8AFoo",
|
||||
"vLg8kVUTmWU",
|
||||
"OK0J0xsuSWQ",
|
||||
"d284rjoU4Cs",
|
||||
"5lE6V4jrUl4",
|
||||
"bS1OpC9xu48",
|
||||
"ZUxpqU_J_n0",
|
||||
"AUliC0c_YIQ",
|
||||
"nJ58-d_viOo",
|
||||
"uIy3b2kzlrM",
|
||||
"vOTLfb0qjfY",
|
||||
"37_7tPZjC34",
|
||||
"MUwCdP5qceI",
|
||||
"lYvNi_ewd8U",
|
||||
"PDG7SzDBTC4",
|
||||
"QHQzoHolWvI",
|
||||
"YmvOEj00LB8",
|
||||
"fLhkXVx7Yw0",
|
||||
"Uj5nhugVgwo",
|
||||
"aHbrDu2o0WY",
|
||||
"yKW6vVgEPCM",
|
||||
"Q0WUgyqQ-4I",
|
||||
"xw3KOPJ9a3E",
|
||||
"5mrTo7c3bCM",
|
||||
"x0ft032W--4",
|
||||
"UdQxEbrIocc",
|
||||
"EHZHAFyv_Ks",
|
||||
"7DojKgPRLVk",
|
||||
"-1CahRD6lJ0",
|
||||
"f3TuYwl7AZE",
|
||||
"TgGTovCcS9Y",
|
||||
"AwyrJGeIN_g",
|
||||
"RbPJD4Wh-l8",
|
||||
"58bFdwYkDzU",
|
||||
"6gIEKGxL7zo",
|
||||
"BeuzJwTuChQ",
|
||||
"YebZyAzLZuc",
|
||||
"9T74b8EQBVY",
|
||||
"TqSVFjw1uNM",
|
||||
"WUYhCmfE1a0",
|
||||
"vRCVUQFUio4",
|
||||
"w2t9JdMNMgQ",
|
||||
"nn6-x9xXMeE",
|
||||
"FDFVPzLlzFs",
|
||||
"qFhvi4bH29E",
|
||||
"qdpsMXLsUZM",
|
||||
"JREk1QeSc9A",
|
||||
"jOVEs-4r1jc",
|
||||
"v4VZI9Ouzy0",
|
||||
"DMg5UUvf-C4",
|
||||
"26J5zRzQkOg",
|
||||
"YkiJ4iJfJbw",
|
||||
"_d8j8nfTEIk",
|
||||
"wMNb3hQXEUE",
|
||||
"1KI8ex22blk",
|
||||
"eCp2Jp7MtKg",
|
||||
"HNlpHBb5dC4",
|
||||
"C6_pDRdIo1k",
|
||||
"ZRWJJHrgP54",
|
||||
"L7jfx4veKIA",
|
||||
"p1a5_qltICM",
|
||||
"nZ0svM8DoY0",
|
||||
"miEzjCJVaZk",
|
||||
"TpC2tDCxABw",
|
||||
"SZmqoqCX__U",
|
||||
"dZ0bYteaI2Q",
|
||||
"GW-cYEzShmY",
|
||||
"iUYuPhutKVw",
|
||||
"97VSm6T2zMw",
|
||||
"jPiQyi5jr1M",
|
||||
"lrUi4nsPZ9M",
|
||||
"rXLeXke-k_w",
|
||||
"mEO4izUqZ7E",
|
||||
"4qI1LjMXfh0",
|
||||
"mVt6myGMa-0",
|
||||
"VJrnhh6FqEg",
|
||||
"vwCeBWbZvrE",
|
||||
"c08fl8-xmVE",
|
||||
"syZ3bvXIrpA",
|
||||
"i3ysNaulop4",
|
||||
"__t3rU0QjsE",
|
||||
"jT32uQQVGvU",
|
||||
"Fzvmh-R2EWk",
|
||||
"QApJSKrLg6E",
|
||||
"L5y3j8SdGSU",
|
||||
"3t8JGblR__0",
|
||||
"e1F90W40t6g",
|
||||
"j6L8UIa5z28",
|
||||
"Zmg2_C1FuVI",
|
||||
"XIXKJXsiBFc",
|
||||
"NVHB1VdWFo4",
|
||||
"-x7-8f7bsl4",
|
||||
"2FSxSiowkGI",
|
||||
"jgFIBIeFve0",
|
||||
"1BLuXK1M5CE",
|
||||
"zmpx6LUT2ow",
|
||||
"z-QUzJTsFro",
|
||||
"0sS4v13DAK8",
|
||||
"gMIBQ0mbRvM",
|
||||
"9kcaEXlTB5s",
|
||||
"ZYArPGMLS4k",
|
||||
"tejW9GMaucA",
|
||||
"z9r4bmXCWiI",
|
||||
"4Zpsb46iaLc",
|
||||
"52Odc8UbvC8",
|
||||
"x-B6-iYDHPs",
|
||||
"8GinSDc0Is8",
|
||||
"5lwbp5d_w0Y",
|
||||
"I4LpTQLzlHE",
|
||||
"lMl6-XH7ozY",
|
||||
"tOubscNbY1Q",
|
||||
"7gb7AH-oWcQ",
|
||||
"mDerqt9Ib-I",
|
||||
"gKV3FNd9WHE",
|
||||
"kH5L1E_r-3U",
|
||||
"sq5TRFxVp0w",
|
||||
"SrBhj7Xusk8",
|
||||
"RxlxEapsz2E",
|
||||
"aCD3_TziiKo",
|
||||
"_HPVZOkxdkk",
|
||||
"pNCjgVQ53Y8",
|
||||
"iPdT5GUoGvA",
|
||||
"cIrlet1oRw4",
|
||||
"1dyWUMBjMwQ",
|
||||
"XZQB3cq18yc",
|
||||
"i75Mn_AeFBY",
|
||||
"_0VwR9WPS-k",
|
||||
"q1JDjgnRdGY",
|
||||
"1JNEpfvRq0Y",
|
||||
"2FGj6HciQZA",
|
||||
"eLi59TtNwHY",
|
||||
"Z3W8OST7tmA",
|
||||
"F2dNYov8-xw",
|
||||
"kPfyJu_CrSA",
|
||||
"7I4Y2HtAwlY",
|
||||
"us-andJDfWw",
|
||||
"Q0Bzn1teWfo",
|
||||
"pADxBfOrYeQ",
|
||||
"KnA1tIR0qCE",
|
||||
"lBNPfBcb6pw",
|
||||
"zt9MIiBfxF8",
|
||||
"FwX12hPqCws",
|
||||
"6e2LXGrze2M",
|
||||
"0dvclhcFjSo",
|
||||
"UTkuDYOIxKo",
|
||||
"4n044cOTiNg",
|
||||
"VxksG15OgRY",
|
||||
"4193MkWdJcQ",
|
||||
"2bDX4Uzy8Wc",
|
||||
"SkyBNX2auEc",
|
||||
"JrSLspuJ2gk",
|
||||
"GZO7V-CWSJ8",
|
||||
"TYMIy4qlRzc",
|
||||
"jK7q_Li7LU0",
|
||||
"UiTIXDQEkPM"
|
||||
],
|
||||
"test_session_set": [
|
||||
"DTGtC7FC4oI",
|
||||
"CIWvk3gJ_7E",
|
||||
"A1fZ6rlOwxk",
|
||||
"cB8WCrmJEEA",
|
||||
"gGQ_2Ank9-U",
|
||||
"b91abcyMeNY",
|
||||
"9MuaFeyczM0",
|
||||
"rwZxhdK_UVU",
|
||||
"hd_WbrqYnTE",
|
||||
"4BWcqA9cFFk",
|
||||
"MuOkgCkob6E",
|
||||
"OdhFWA3Fgb0",
|
||||
"Pe3ZabrUiUk",
|
||||
"U0c_uv8DU08",
|
||||
"kTqt_3YJBbs",
|
||||
"eWXVVMi4bf8",
|
||||
"RCD-vXumMIs",
|
||||
"DuwiezJCZ3I",
|
||||
"BNgEqjAiKU4",
|
||||
"Cbn0t7eqaEs",
|
||||
"D3M9Q0WwuFY",
|
||||
"oO-Bv4iRZKI",
|
||||
"bBvqtbZdSZQ",
|
||||
"-mnoItCi32Y",
|
||||
"yQPxKfz-bwI",
|
||||
"ITVZ4cx5big",
|
||||
"uY5_e0L7mjw",
|
||||
"J_Pl9ftnNXE",
|
||||
"Jt_pb40MRy4",
|
||||
"E3hAj2niRNo",
|
||||
"kl2xVSxVECQ",
|
||||
"gzebuFs90Xg",
|
||||
"6DQSIzJ9J-g",
|
||||
"pJJlnryTYWg",
|
||||
"X87qsfVLFoc",
|
||||
"W7BIPWU1WHc",
|
||||
"Oj506EVOiQc",
|
||||
"DPEvWvcstxA",
|
||||
"Pk16UT5-zeM",
|
||||
"XMK1KEk5OdU",
|
||||
"PpNm5llsEW8",
|
||||
"sAp3rvablGk"
|
||||
]
|
||||
}
|
||||
}
|
||||
479
third_party/EgoSpeak/LSTR/data/data_info.json
vendored
Normal file
479
third_party/EgoSpeak/LSTR/data/data_info.json
vendored
Normal file
@@ -0,0 +1,479 @@
|
||||
{
|
||||
"EasyCom": {
|
||||
"data_root": "data/EasyCom",
|
||||
"class_names": [
|
||||
"background",
|
||||
"camera wearer speaking",
|
||||
"other people speaking"
|
||||
],
|
||||
"num_classes": 3,
|
||||
"ignore_index": -100,
|
||||
"metrics": "AP",
|
||||
"fps": 5,
|
||||
"train_session_set": [
|
||||
"Session_7",
|
||||
"Session_11",
|
||||
"Session_10",
|
||||
"Session_6",
|
||||
"Session_9",
|
||||
"Session_4",
|
||||
"Session_8",
|
||||
"Session_5",
|
||||
"Session_12"
|
||||
],
|
||||
"test_session_set": [
|
||||
"Session_3",
|
||||
"Session_2",
|
||||
"Session_1"
|
||||
]
|
||||
},
|
||||
"EGO4DSHUFFLE": {
|
||||
"data_root": "data/EGO4D",
|
||||
"class_names": [
|
||||
"background",
|
||||
"camera wearer speak",
|
||||
"other people speaking"
|
||||
],
|
||||
"num_classes": 3,
|
||||
"ignore_index": -100,
|
||||
"metrics": "AP",
|
||||
"fps": 5,
|
||||
"train_session_set": [
|
||||
"5b8c91ca-9d16-48b7-acce-4dec221ffd71",
|
||||
"0232fbc8-c9b1-405c-bd42-a9d773fd4835",
|
||||
"39c74452-1957-40db-894a-b39df51e1437",
|
||||
"31488ff6-05ef-4362-9786-35981b06f307",
|
||||
"4decc0fa-e98e-4bf9-bbd1-d9b852b04830",
|
||||
"efeaa284-2e22-44b8-bdd6-1bb4763e5b35",
|
||||
"07f1486c-44a7-4c69-9532-720c81c99eb0",
|
||||
"19c28cbe-21f2-4b40-a6e0-e9932b5f0605",
|
||||
"1c5c0b52-1287-4df3-aa75-8ccc916b1c10",
|
||||
"e92255bc-1be6-4ee9-8f8b-d1336a0e409c",
|
||||
"307e8837-1fda-4cf5-bab5-d16fee20f84f",
|
||||
"bb050b8a-195a-4b50-b5d0-1ed535226a39",
|
||||
"ca22f118-2472-4df0-a9cd-3d70424b211b",
|
||||
"08b0935e-6260-4bd6-86ca-f6fc54e388be",
|
||||
"666a21c0-7469-436e-868f-64eb8022f64f",
|
||||
"77e09fd6-b560-4439-addc-a1a8a3122046",
|
||||
"f00bd659-56e0-41dd-8eba-c5fdb45e7232",
|
||||
"c2c7f072-a02a-480d-8342-b43ffc314db9",
|
||||
"ed72b048-e309-4510-b049-b0288ce4478a",
|
||||
"56c5af79-f9d4-478d-96ef-6d71e0bbbdfe",
|
||||
"3d8acf67-fad6-421e-841b-82b28c82f7f6",
|
||||
"977ac980-f908-4c29-a2fc-dd14ec21c0bf",
|
||||
"08742cb7-bb6d-47ae-af5e-3f1f306ccda6",
|
||||
"f3afa5a1-8a97-4a7d-945e-1cc70931e44a",
|
||||
"65172b19-a246-4dc5-98e9-1b526b00ac89",
|
||||
"ee1893cb-2ff7-40e0-af8d-19518696b698",
|
||||
"53144267-6da5-40d3-91f2-eaa85be367de",
|
||||
"26080fb8-0588-449c-a7f2-2e0162507e02",
|
||||
"c0ba997c-9ec6-4e83-ad50-a88dd677b881",
|
||||
"6640a311-86c4-4de7-964c-e3a4a67a3e02",
|
||||
"5f1fcf84-4001-43ec-ad88-9e06ea2f663d",
|
||||
"ea5a3f9d-7b21-46d7-a17d-bcbd4955bae9",
|
||||
"eb3675b8-f6a8-4ea9-82df-1080eac5b093",
|
||||
"023aaf52-fbe9-4b61-af69-ed33e91f3717",
|
||||
"86af705a-a326-423a-bdd9-c6734804c30c",
|
||||
"55363f67-1b41-476f-a83e-7e9741786ad7",
|
||||
"ce8ea1a3-65e3-4ef7-abeb-2b0bc1c84773",
|
||||
"d3b7e314-f48d-4888-b496-e1eeb252cd8a",
|
||||
"59000084-d286-4c83-90c8-2fbbf80d2d31",
|
||||
"a47c590d-66aa-4bd1-b552-f40951ba9bf0",
|
||||
"61c73721-9476-4ac2-a987-ecaf9a04dca1",
|
||||
"0124376a-720d-4265-8c8c-5ff47f04d579",
|
||||
"f62a8100-7ed2-492d-9709-064514d552bf",
|
||||
"13d71019-9093-442b-bdb4-c79310150592",
|
||||
"74e71420-9bd6-4337-9a7a-a3dcc4cab7cf",
|
||||
"383466b5-cb00-4e76-9e6a-dc1b0c9b125c",
|
||||
"9ef6ef95-6b97-4c78-866c-641fc7d791f5",
|
||||
"ac7d3d16-d74d-4fe3-a1e0-cab0b1adf492",
|
||||
"c46cbba9-2cb3-4d84-be18-71af5539ecc5",
|
||||
"5eb74ea5-9b0b-484c-82b2-ee86479c5d07",
|
||||
"6f80f9ae-c276-413f-accc-fad74efe8a95",
|
||||
"73f99cf0-d997-4488-b6a6-993934a9fad3",
|
||||
"fd50b1d6-8142-402f-88c5-7c44ca215081",
|
||||
"51fee605-d077-4d78-babc-9bbcd8d898ca",
|
||||
"5e3c6abc-599b-44f8-8401-d5720627bba2",
|
||||
"91151b86-65b7-4854-97b0-9c5b93a251ed",
|
||||
"5be9a2e2-5efc-4fc6-a65e-806e0347e19a",
|
||||
"94ba75fc-8d7f-4bb4-a756-ded62a4c1761",
|
||||
"c925c38c-3fe3-4eb3-9944-3fba3eaca4db",
|
||||
"d97bedc8-72df-43be-a55b-4da1ae42dfd1",
|
||||
"e60b110e-a9b6-42b7-8f92-082cd74c4937",
|
||||
"3627996c-8188-4486-8feb-938b9b0cce2b",
|
||||
"d2b674ca-9f02-46ba-820d-f0b52c76749f",
|
||||
"2b90b963-e829-4787-8fcd-1a1a434ffdfd",
|
||||
"ba51db0f-2bf1-4746-894a-7db0ba54e357",
|
||||
"6c1bac5c-aead-40b8-9c17-be97d86b68d4",
|
||||
"63483ea2-d569-45fd-82b2-f80836a8b766",
|
||||
"c81eb608-6844-424e-b8b7-524ff652cf3a",
|
||||
"cae5d4bf-bbf9-45dd-876b-a48579782348",
|
||||
"7813b44f-04ca-4a53-868d-fe185b615b68",
|
||||
"1f12c871-9b3a-4611-a2b4-9c39059052a4",
|
||||
"70316529-ec02-4d26-bc68-4b436bf0030a",
|
||||
"6d367aa4-56ef-4932-9d48-e7530e717ea8",
|
||||
"2e651150-9338-4103-b231-2f089aa19df7",
|
||||
"d2cb4045-8ef1-4288-8398-ab6d4d6a0709",
|
||||
"3e4b9c29-06d2-4ade-a2be-bdec028bc68c",
|
||||
"4f3c8276-99e0-4c99-9e61-5d7c06332b09",
|
||||
"12634773-451f-4c82-b915-265c703ad737",
|
||||
"5f615b11-2ff8-4ff1-987a-1cd61ea78dee",
|
||||
"de96ee14-be42-471d-b110-aabc948260fa",
|
||||
"8504aae3-4f25-4fa4-88c1-beb3fba83f13",
|
||||
"d2c0f78a-978e-409f-a78f-3e40b3431909",
|
||||
"0f846b5d-1cea-40b8-b18b-dc6c54544a67",
|
||||
"37afa156-a068-458e-8bf7-ec6f1f2415bc",
|
||||
"2d205d38-f79c-43f2-8c39-ede697065ab2",
|
||||
"36e7b2ad-dddd-42fa-a427-6411ff12f9b9",
|
||||
"355f8f4a-cde2-4108-821a-5ccea1d297fe",
|
||||
"c37e680f-03b1-408b-9b09-1b2ae733951d",
|
||||
"6958467e-388c-4f6d-a8c0-d4782166daac",
|
||||
"3051e028-71f8-4f3d-a72a-010934c3566f",
|
||||
"fc0f4c90-a1ee-4147-990e-de231394f484",
|
||||
"c2af1389-b165-42f5-b4ee-c90402d7006b",
|
||||
"1acfcabc-653a-4c03-adf7-64838d09a7b4",
|
||||
"3b79017c-4d42-40fc-a1bb-4a20bc8ebca7",
|
||||
"6edb7036-3369-4c03-8e3a-439ffbeb4724",
|
||||
"2ac543c6-b52b-4219-b2e4-4130078c0673",
|
||||
"b3ef3563-ecc0-4a15-9a7b-4feb4558d953",
|
||||
"6f001eeb-fed5-43e4-bfa2-84e274cbe5cd",
|
||||
"1ce2252b-31b8-4536-9902-0ba7fb039340",
|
||||
"c2413391-7c1b-4fd6-8b1d-98ee7888b9f8",
|
||||
"a4d53159-e2c0-4a47-9e78-af9bc9291f0e",
|
||||
"66ddb7ae-90d6-4a58-9f3c-7f1d2604ecb0",
|
||||
"290fc9ba-0a29-4eed-b0e0-64edc5bf54ea",
|
||||
"e3a53edd-52ff-4608-a80d-77ecfbb0f961",
|
||||
"50895f7f-5667-4b43-94c1-d7b4b9e238e7",
|
||||
"3aea2d62-0340-4c6a-8019-3cce9117a65a",
|
||||
"63ab0246-2f1c-4621-8b71-75075b30d852",
|
||||
"228eb504-c1b3-43b9-818e-2f7e539a259b",
|
||||
"02de9500-d574-446e-95fc-3e82c367385a",
|
||||
"bd03445e-9016-423a-bb48-cd998836bbff",
|
||||
"a16cb772-d0e2-4654-96d7-372c21484b90",
|
||||
"0fa75089-7b28-4988-8664-a2c93542ffb0",
|
||||
"fcc6af87-918f-4d0c-a0a4-8aab36891fc4",
|
||||
"b9d7642e-2ced-4589-b7a6-69ad54570c4c",
|
||||
"ba5b1882-c9d7-48e7-85fe-2c7b10494fac",
|
||||
"92e0925c-566d-4a67-b93e-3e4b57f9b3ab",
|
||||
"130a49d0-0315-4eb0-a752-a4533bd880a9",
|
||||
"0fccd442-a901-4d77-aac6-dc4276cd6591",
|
||||
"49c27d75-44fa-4947-899a-8d0511a10416",
|
||||
"e2ffc7c9-c035-4709-97c6-3764499188a5",
|
||||
"2069aea3-ac99-4b70-891e-1bfb8b15aeea",
|
||||
"cdbc7718-4cb9-4e78-bbde-3d9614f8dcd1",
|
||||
"32464967-b0f2-4b2e-a57c-7f774e7056a7",
|
||||
"ae6466f8-1a53-4cb3-8718-96f78f391b4b",
|
||||
"d66c7c66-2b5f-4e7f-bbb2-f0adae68dfcc",
|
||||
"da84b37f-ec6e-414e-b52e-1231104a68e0",
|
||||
"19866484-b3d8-4619-85b9-e900de7fa80d",
|
||||
"c4392ca6-4706-4a96-a6b2-29e6ef544500",
|
||||
"6798a65f-d79a-413b-baca-115f09caf0c4",
|
||||
"b844cb2d-6428-4d5e-abac-958f2704d4c8",
|
||||
"8e9c0b21-2230-4a0a-b35b-a198b90abcaa",
|
||||
"3492c684-74b5-4317-9e5c-320f3ea01899",
|
||||
"5beb520c-d413-4998-bd9f-7960a2baf744",
|
||||
"c96fa96d-d23c-444e-bdef-481317308bb8",
|
||||
"502ce8f6-bd9e-4e40-a5c9-40bdce36b87f",
|
||||
"221fd24b-ba2e-40d5-a0c7-a879ece6b34f",
|
||||
"fb62dc3f-a1cd-4f0e-88bc-504fe9d8585d",
|
||||
"e9acbb7f-c45c-47d7-9dd8-16194319fc97",
|
||||
"0d4efcc9-a336-46f9-a1db-0623b5be2869",
|
||||
"84a388b6-2844-421e-a176-84d29958a76c",
|
||||
"3ecc3e4d-5466-4243-acca-2feea47e09a2",
|
||||
"e7b6b4ee-3f2a-41cd-bc18-c7e993cb413f",
|
||||
"5998ece4-98cd-4d5b-9104-1517d13728bd",
|
||||
"c9f84bf4-108c-4151-85c0-e87375c53b91",
|
||||
"5eaa3bfb-cc71-43fa-88dc-d367ef929a7b",
|
||||
"0fe736da-48f1-4961-bd7a-b75ce13c91b4",
|
||||
"2c7796e3-b1e9-413f-a0e8-8225ac7d586e",
|
||||
"69006355-3fb0-443c-826b-ea8c9d27f70b",
|
||||
"8d73dd20-5349-477f-9630-218e6b08bb95",
|
||||
"e7633ad6-e406-4a18-b1ba-6c834ec9c047",
|
||||
"3b3a4043-3d74-4575-8aeb-511c4a879241",
|
||||
"5daa132e-0b97-4d0f-be9d-b514285661fe",
|
||||
"52fb1a73-2ba7-413d-8596-33615daecc73",
|
||||
"9f3cb8b5-5d62-4236-be5e-8f16d59c9f50",
|
||||
"efdc3dc0-275d-4073-9179-32b14b224654",
|
||||
"683635b7-5a1b-467c-8d4c-c34e8bf5fdf5",
|
||||
"2db7496c-aa43-4d9d-8df6-404d2d9bd810",
|
||||
"9f3edf40-3dca-4086-b995-4985f6848a14",
|
||||
"ef37e1e6-5be3-4e8c-9f70-c84f341f9aea",
|
||||
"5df66408-7a8c-43bb-9235-d088aa3d19dc",
|
||||
"453f43e9-c1d1-4749-9720-73546287b82f",
|
||||
"1943bfb1-ba78-4a6c-ade7-11e44314dfb4",
|
||||
"fd8cc20a-915e-45a9-93df-320ad998aee4",
|
||||
"bc6cf777-fd8a-4427-9867-92a443826827",
|
||||
"812f0590-4ccd-4c9e-8978-c45e6debce55",
|
||||
"e6f4e36f-96b7-48f5-afd1-27cc408bdd07",
|
||||
"1b6dfc9f-c873-48c6-9f25-a2d490e269cf",
|
||||
"bdf1ca5a-321e-4d3c-ac9e-da2ce1f5a69c",
|
||||
"604bf883-ce55-4c59-bcc1-3ceea1128211",
|
||||
"679f2016-4802-4bf8-a830-d1f45fdfcd71",
|
||||
"6f3f70b4-8428-4e5f-b1dd-86cd55d8e63f",
|
||||
"83d8be38-ce02-41b9-b444-7291bad513d7",
|
||||
"7fd21591-6a33-49cf-a32f-f8dfdf5ae697",
|
||||
"a3dbec39-7126-4949-a916-eca55fee2b49",
|
||||
"03cebfe9-3319-45b5-aba5-e7db10ea6fdb",
|
||||
"ff9fe54b-a959-4d7e-a6fa-93e311a68168",
|
||||
"172e1afa-5581-455d-9a27-590e9a1706c2",
|
||||
"3ab3b82f-bfa8-4b39-a9a9-e397341ae572",
|
||||
"b171dcaf-94f2-4bcd-8695-b2648bed60e1",
|
||||
"063f5040-9e94-42ae-b0e5-b13413e0fe55",
|
||||
"136cef9c-e17f-49f9-ae45-a17e7e5c5477",
|
||||
"f906b093-281f-4872-8010-f1f3a320ee2a",
|
||||
"33365c93-7f1d-4a7d-a9bc-b33010aeafbd",
|
||||
"e98405ee-0a5b-4df2-bb7e-97729b8b3c11",
|
||||
"bd13e435-1e38-4b22-bfd0-b620fc74cc8a",
|
||||
"7dd0eab1-b83e-4bb4-ad91-7bbdf4592647",
|
||||
"5f5ea963-4f59-4705-95fe-4a98c9fcd12c",
|
||||
"7e6dfd31-8544-4fad-9e49-0f05516cf8cf",
|
||||
"f817b176-2dce-4858-999f-fe62aac25e2e",
|
||||
"83553642-1a38-4c54-8154-c4cd4fd81173",
|
||||
"dfe025f3-573e-4297-b17a-30e990cfa310",
|
||||
"2091513c-8f4c-4817-8c22-cdbc6303a331",
|
||||
"d15eb34f-e425-4990-aa2d-2dc454fb4b20",
|
||||
"0e519838-1e4a-41fd-ab30-34bf745a3339",
|
||||
"c920e557-141f-4556-95ac-f494ecc0c84f",
|
||||
"91e97b3c-c912-401a-ab99-da01b90682aa",
|
||||
"d6adb1ae-f689-438b-ab81-8a2e37f9654e",
|
||||
"cd3092e1-6022-4a11-bfc3-a4bb80487ece",
|
||||
"a874ef10-022c-43e4-b4e6-e01d8fcec988",
|
||||
"2e6ff051-0037-4001-90a8-de7643a96f08",
|
||||
"134c4387-77d5-4ee0-911f-e50e64499b42",
|
||||
"421f1c8b-5237-4735-9003-82021ba651b1",
|
||||
"9ee0474c-615c-4383-a047-8432b8b24784",
|
||||
"78efcc13-3718-4acb-acee-8157a54501d6",
|
||||
"961a8de1-0d33-44ab-a9d5-3d92de13fde4",
|
||||
"49e25cbd-1aaa-40dd-901b-bb3e524ebcc0",
|
||||
"5a5b8c37-e3f6-48f2-96d3-289a043eaee4",
|
||||
"394d1df6-bced-4f04-8c2a-a1ea3a04b984",
|
||||
"b87c3e0f-a5af-4663-a87d-d0689d73343b",
|
||||
"09194996-ff4b-4a4f-9800-e8aedc9be7f2",
|
||||
"3f920762-f4a7-4bdb-86dc-05a3c3c8e875",
|
||||
"5204624f-90ac-47bf-a71a-486d43f4351c",
|
||||
"df24897d-a478-4fbb-816f-1d14c59da30d",
|
||||
"9ba7c2a0-c86f-4967-9284-9be88ad3eacf",
|
||||
"3f28dd4f-f653-4937-83b6-9f80c1284a85",
|
||||
"b2735994-59ae-4478-9c5f-93f5ccf18c8b",
|
||||
"5e965191-bcd1-491a-a9ce-e014e970020b",
|
||||
"85718cf6-e4ba-44e3-880f-143c627f12ff",
|
||||
"d0af657d-9c21-447c-a4f9-199b38deac5b",
|
||||
"f8289825-2858-4dfa-b173-f4c506359158",
|
||||
"8f2d26e6-239e-492f-aa1b-472f8ec85050",
|
||||
"836be9c2-cee6-4fb9-8bbd-74197d213c5a",
|
||||
"cd998593-e5bc-49d0-86a7-333acf9abf93",
|
||||
"8deeeb88-19ed-4747-9c79-d94104bed7e0",
|
||||
"6d1dc078-8b04-480e-a36f-298594ee9592",
|
||||
"83464c62-929f-4bf7-9a04-43071189edde",
|
||||
"1efd8b7b-65f9-4e73-bd49-d707190d6519",
|
||||
"f0b69458-25df-4dda-be8b-e14383a57a50",
|
||||
"88260ff4-be0b-4544-98ef-78b6a0d61ebc",
|
||||
"39fb2989-1dd2-490d-b793-dce86c36b43a",
|
||||
"b4f650b4-53d6-4069-a55f-43f2de7ff47b",
|
||||
"15b6759d-2370-43ee-9a61-09e453db2469",
|
||||
"2e4e0dac-ab21-49ff-9563-e320acd73f2f",
|
||||
"986c4e43-ac2e-4983-8d56-f1dd34775133",
|
||||
"12c1b77c-cbea-4c20-8b64-7154a25bbbe1",
|
||||
"a3da62c3-3f46-499f-a4c6-acf93bd1c63d",
|
||||
"57b6455e-2acb-4076-b8c4-1f2cdae97185",
|
||||
"d6d8261e-0ee0-4002-bddf-e94fdc1075c7",
|
||||
"56683d7b-7c65-4f42-b5a4-818f3cd5224c",
|
||||
"9090779d-803e-4fbd-9166-d9e90ff2d36f",
|
||||
"d5374572-5576-4f83-99c8-a0b12c4de160",
|
||||
"ffddab3a-dbec-49b0-b114-5a1e3d805efa",
|
||||
"38c02527-1b2f-458b-bad8-73a282c1d99a",
|
||||
"d612edfb-e904-4de0-abce-a5fd9ecc449b",
|
||||
"1261ee8c-80fc-4a3f-9dd3-29f4a918a9fe",
|
||||
"8e860a91-0205-447b-a706-aa894d23d3db",
|
||||
"fb07f205-868e-44c9-a704-305daaf03301",
|
||||
"48f7028c-b6bf-4ca5-81e9-128147fd02c9",
|
||||
"c6a6e1e4-24c4-422d-9d18-f062240ddf23",
|
||||
"ba280b86-22a2-4509-b16d-b55e04680461",
|
||||
"f0cb79ef-c081-4049-85ef-2623e02c9589",
|
||||
"6dbfc053-7899-40d8-9827-0ccd21f3ee0a",
|
||||
"50d3035c-2503-4bcf-b28b-e66c92c98f09",
|
||||
"41f7665a-a692-42e0-b28d-9769c03ab99f",
|
||||
"cb1f70b4-0bf6-47f1-bc98-2e9365a55c3f",
|
||||
"7f566780-4ed2-41c5-9faa-06759495faab",
|
||||
"0e08e1c7-3add-4e41-b28d-cd5f4413a422",
|
||||
"672e0d75-df0e-4715-8c42-f1131b059a6c",
|
||||
"8f15aab0-9c42-452e-b78a-47a48da1f3c1",
|
||||
"edaa0b1a-d504-4852-8025-4f12b4d293ac",
|
||||
"4a9ada82-9f86-495e-b17f-25b39bef2297",
|
||||
"07774a7c-c5ed-4133-991f-926a316068a4",
|
||||
"5a009338-0aff-4313-b02f-c48630ce1702",
|
||||
"8324c0ba-7a10-42e1-90fd-afb4d3afa789",
|
||||
"3db37314-adaa-4a4e-af11-c9190fa6b65b",
|
||||
"b21b35d2-6d69-40a2-8c36-aa27e4a980ae",
|
||||
"6daa9124-d894-42e5-8d98-2e0e6d1191ac",
|
||||
"8a256d7f-2ea1-4b0e-94ea-168a1ed85678",
|
||||
"08b6f5b3-4471-4cfd-a95e-4901cb1ded57",
|
||||
"ea9d85e7-6a9a-451a-b019-972a6891c11b",
|
||||
"f84fd67e-0797-44f4-b3ff-78da646362e2",
|
||||
"ad1a9ea9-d1ab-474a-b715-1a42d892ef5d",
|
||||
"ffb191f8-894a-40e8-baa5-5578aa6ba706",
|
||||
"fd394f00-4049-49e5-9c07-ee3e86979003",
|
||||
"9767a3ca-0a9c-480f-adfe-e4164c52c760",
|
||||
"95cdb269-0088-4451-9a0d-f72349081fab",
|
||||
"4ade0a82-16fd-471f-9fcc-fd2724887aef",
|
||||
"fe69a78e-7773-45d1-9e0f-bacee52dac83",
|
||||
"3aed7bbd-75f0-45ef-b9cc-7f5ff6f7fbbf",
|
||||
"f8d71e9a-982a-437c-a968-867149c24eb7",
|
||||
"8351998e-ead7-4bc1-b0a7-a7209da92b3c",
|
||||
"3eb00a40-880a-4ad9-8cec-3ba0245ec472",
|
||||
"602b224a-f5ac-491b-88e3-49043f362b6e",
|
||||
"89125bc3-a413-44ee-a7e1-2753c511ecab",
|
||||
"00792fa8-988c-4c85-8e80-73eb3ac53e80",
|
||||
"094d1ab2-f34e-4a2a-ac3b-8671ce5faf65",
|
||||
"4cf10990-13e5-4a1e-a913-7ec4b8c8ab02",
|
||||
"a9b191de-d565-4981-8f68-02069a2ba16c",
|
||||
"0eeab9d2-a55c-4708-94c3-cc1e08f9f262",
|
||||
"4e78f728-06f6-43a3-ac8c-c0cb159e5b32",
|
||||
"be99e9e2-82f7-4f1f-aff5-371834fde0da",
|
||||
"5682f9aa-bdfb-409c-b4c2-c0105b35ee78",
|
||||
"3ce15244-5293-4a5e-8c92-52ab29c6949a",
|
||||
"b4b2c55b-c107-468f-bbdb-f73a0e4dca34",
|
||||
"8a409551-657a-48c0-8beb-056c228ebca5",
|
||||
"3c17a762-c839-492f-b361-009c1187ae4c",
|
||||
"22172001-0e17-40db-bcce-825e19532297",
|
||||
"5b99234b-b940-405d-9123-70218bda24b2",
|
||||
"f7869fd8-9af3-493a-a627-a3e95e7c3577",
|
||||
"057e9f41-e6ed-4575-906c-aca9413b35a5",
|
||||
"70e88e82-5c06-44fd-959c-d4b2d40999cd",
|
||||
"b95bd349-11bb-4ade-9cc6-3be8dcea06dd",
|
||||
"19b880da-910e-40c7-970e-cd8718f2faf5",
|
||||
"e05a2182-a97f-4d0b-87e4-66b0dd405e58",
|
||||
"3072481a-04fe-4959-8410-018a41a91df1",
|
||||
"fc2b2014-9dc4-4a5d-8a1d-25a6911bff7c",
|
||||
"db1d3def-397e-470f-811a-85c4cc5338a9",
|
||||
"ff283659-4a79-4642-8078-4366df38e1f1",
|
||||
"bc8bae7f-ad9f-4f72-ae5c-b3e7ca8dd8f0",
|
||||
"a3bd1be3-edfe-4f4e-89a8-dd56ce6ece08",
|
||||
"2360312c-4347-476a-990f-2633b489fa7a",
|
||||
"b9e32e51-8c18-420b-a582-200c79b8208d",
|
||||
"0511dce9-c126-4a11-92ba-8082a4ec89bd",
|
||||
"224d8b17-7eaf-4529-9b75-c6ce3078d640",
|
||||
"10732476-f045-4feb-8199-e4e04685f7ca",
|
||||
"5753e3fe-f1c5-426b-9f45-5a8efa6b5370",
|
||||
"b69564f5-5e5e-4097-aebd-9f52f211ff75",
|
||||
"97eb574d-d8bf-44a8-a6ef-1204b789db43",
|
||||
"85506322-449a-45c0-b77a-48a8077b4bbd",
|
||||
"ed65870d-ba08-4894-9cad-7f34e2c6f9e4",
|
||||
"cf682058-962f-45c7-95ab-dbbd2f87acc0",
|
||||
"65229ac2-9033-4002-af6c-492fe49bc563",
|
||||
"4c85a780-8a8e-46c9-a0f8-c7e9703ef998",
|
||||
"1ac6e060-a6d2-4b42-afef-599be78a533a",
|
||||
"a22beaf0-377c-4b95-b472-867280a525ae",
|
||||
"f5bc6a45-1591-42bf-8ed4-8286382b31bf",
|
||||
"06592f50-7f29-4ca9-94a9-1b6c389d50ea",
|
||||
"4ca3d9cf-4972-4e1d-aca3-28b036d6bdc5",
|
||||
"076bdb81-5c75-4282-9f3a-a387624575f3",
|
||||
"a5fe8210-1c37-4dd9-bd27-74051c521c3d",
|
||||
"c1ca7c5c-2473-4b83-a507-7d187609111a",
|
||||
"7c4079e7-4d68-4b0d-b015-5971a9866c60",
|
||||
"ead54fb5-dee9-452f-8145-227ccd2405e2",
|
||||
"7c776700-1d6b-4ee2-8155-b7f6855afceb",
|
||||
"71a7456a-da6e-4e38-86ec-76bacaa6db5b",
|
||||
"26c042d4-bf21-4d10-81f9-303d3816c114",
|
||||
"c7820a4f-55c9-44eb-bf7e-b965a9d0e924",
|
||||
"5b5b6b8c-9b88-4181-9ebb-14abb7783e1e",
|
||||
"db12030d-7577-435f-9d48-5c1aae615c4f",
|
||||
"e37c82c5-483f-4e68-8bf3-2a22dcef0b1e",
|
||||
"0b4cacb1-970f-4ef0-85da-371d81f899e0",
|
||||
"3949f121-af2b-4c7c-8a49-07befc08abbe",
|
||||
"73c1ac5a-80a0-4e55-b143-52df5697c9ea",
|
||||
"0f325614-48c9-4a6a-b7f2-731c26ddeb97",
|
||||
"572117cf-a76b-44d2-804a-4901bfc545f4",
|
||||
"33fd4f1f-fb01-4d3a-9b8d-b8d8024a2dab"
|
||||
],
|
||||
"test_session_set": [
|
||||
"66cb6d16-f972-40e9-88c6-64b6816b900c",
|
||||
"e62b75a0-f1f6-4e45-b3db-17fd7ff5756c",
|
||||
"13ffc0e4-eebb-4baa-97af-5983ce46779c",
|
||||
"d062849b-c60f-4369-a113-50f6ce8521fb",
|
||||
"bb7f1e75-65ee-4d98-adc8-c70551a35940",
|
||||
"7a6576d8-007c-4ee9-b7cc-8cc2d020e1bf",
|
||||
"4ce65f3a-21ee-4f77-85c0-b3ddfe1e6381",
|
||||
"783d7043-b27e-4744-a969-5f40fb40827c",
|
||||
"81b74694-a2e6-4be1-b9f6-69f3aa03f50d",
|
||||
"e6317c78-06f0-4c6d-9886-ca67712772ed",
|
||||
"44bcb1fa-9e64-4bae-927a-4d5d53e536a6",
|
||||
"218e79aa-3f56-4bd4-aa2a-14027bf7d52d",
|
||||
"561fd770-f9a2-41a4-90da-beeb7d96ee7f",
|
||||
"a2d94b61-57c7-4ec2-ad94-94b3585a6b64",
|
||||
"764a08dc-5ac5-4c09-a281-0210cdd5a55e",
|
||||
"c1057c9b-331f-458b-8914-2703511e8b13",
|
||||
"3edc700d-7d14-4154-a3e7-5b0e8c3fde07",
|
||||
"1492c03c-1887-4576-8889-5bc2848967de",
|
||||
"2ea37ccc-e1ec-4e0d-a2a7-9707ea300b7f",
|
||||
"65cad708-34b8-4158-be96-b88710288473",
|
||||
"be8f4445-3958-4fc4-8cdf-b14d1d0eba73",
|
||||
"3c62a0d2-10d7-4a9e-ac2c-02505663afad",
|
||||
"be25331c-b1a0-4187-93fb-3b59437258c3",
|
||||
"108cebaa-777d-42dd-8b1e-69ea7f443a0a",
|
||||
"be3b65c8-ba74-4321-99b4-ca0e99e6fc2c",
|
||||
"3bd74a94-ec3e-4138-b78f-ad76af4f07e2",
|
||||
"4356f6d5-576a-4999-af40-744c242fff22",
|
||||
"b0c57748-65c8-49d7-9681-ddcfcde3298b",
|
||||
"12706b2d-0a0b-4511-8feb-ed8065746e50",
|
||||
"f0e9a139-86f6-41bc-82f1-11283954a781",
|
||||
"bb3c6009-892c-4071-9bb4-692b83ed12f3",
|
||||
"90bf3f44-60ae-4430-9dc6-2365f7901fc5",
|
||||
"2b5cea2f-b019-4ecf-abbc-b2346bd65a10",
|
||||
"8e9f89d2-f02a-4c75-8831-6785ef6bd7f5",
|
||||
"13c24739-e514-4fad-8288-6686bffa71ad",
|
||||
"1828b207-9601-4521-8c0b-5d9e829fca78",
|
||||
"0324c3b5-5154-4de1-9869-eb866b568e0f",
|
||||
"1a746147-2dec-4a37-a51b-c94e4f1de010",
|
||||
"3a20bf0a-6eb1-47c5-9bf8-8445446feb61",
|
||||
"c2a8f168-06b0-4766-8865-27f08cb1bcae",
|
||||
"f55f2dfd-704f-42cf-a39e-72e2560314b3",
|
||||
"4ea88cb9-bff8-49e0-a855-cc1902db9a99",
|
||||
"4e0053f8-566c-4952-abbe-4ea877782a0a",
|
||||
"80ee109f-b097-4755-8a39-9f0b33adb170",
|
||||
"933ddf59-52d2-4d4c-b653-05de44ded245",
|
||||
"b432e10e-cff1-4f15-a8fe-1e2350e94b36",
|
||||
"157c8e81-117a-4a15-a047-9a574181d44b",
|
||||
"cfc6da55-43fc-4cd8-96d0-9de5166f4a63",
|
||||
"aabeb7f9-5ada-425a-9545-c98dc443887a",
|
||||
"db9978fb-fe4d-467b-9495-ab7daf3b1b55",
|
||||
"6549c35b-8b1d-4577-9434-ac808e4d7057",
|
||||
"e50126ee-f4c1-41a6-a2e8-79636b40b341",
|
||||
"49a7e53b-aba5-4e53-b3bc-1492657fc8ae",
|
||||
"7abe6d4a-f594-4a15-af08-ddd591ff500c",
|
||||
"03c50078-a08d-43ad-a3c9-87916aebd865",
|
||||
"a9a0ed92-fab1-4b6e-ae27-714692464641",
|
||||
"3c3207e6-57b9-4be1-96fc-5b2168ae3cd3",
|
||||
"895545f1-f194-4c0a-8513-87d386670f15",
|
||||
"7b76be35-4e6d-42d7-9fa9-14a21fc327cf",
|
||||
"e4670e4f-0b1a-450a-9778-9ee2409db554",
|
||||
"34bc808a-4b98-42c5-a1ea-6f565dd4aa20",
|
||||
"bec4e063-9be1-402e-a7b5-dba20730c933",
|
||||
"20a3e8e5-3f43-46a5-84fa-6888bd9c5c88",
|
||||
"5da4efa3-8e54-46dc-a8c2-3016fa4e12f8",
|
||||
"0d998741-de68-4006-898e-7108de4167b1",
|
||||
"4d144ff3-a853-4b79-9825-d7fcea248373",
|
||||
"dce1d8b4-2c29-4fec-a343-77885899c434",
|
||||
"dab038d7-9e08-4edf-8d03-ee7e2af5a10e",
|
||||
"a1055434-9e9b-4d69-bac3-374a39f801da",
|
||||
"e7086057-b5f7-4286-87c6-ab110072c5b0",
|
||||
"3af65596-1aa5-4126-933b-b98ccaccedf2",
|
||||
"19f880ae-9770-45a5-93f5-00047f22d5f6",
|
||||
"36888bbd-1ece-4a3a-8aff-b0f6ef905752",
|
||||
"f42a7e14-c3ee-4b05-a59b-8d349f8bd19b",
|
||||
"d36d9514-a99b-4534-b54f-3fd98a19f0ea",
|
||||
"6c4f7cb0-caa0-4a35-9071-a84c9ec2ab2f",
|
||||
"666d9243-f4f9-47f5-8fba-cee8568abf41",
|
||||
"2f14e742-dc58-43c8-8ebc-7244487dbba7",
|
||||
"f8accf9d-4427-4458-88e9-89d51a3f5136",
|
||||
"50810487-0b0f-45e0-8f37-515644189e81",
|
||||
"f0445727-2729-4a76-9434-fb081a14cd9d",
|
||||
"f55f890d-fd90-43f6-a11a-a1920c21890a",
|
||||
"16851d4d-35e0-49b4-b3b5-28e67b564db3",
|
||||
"5c8f8839-4bf9-45e0-beb0-6c17a5ead0d5",
|
||||
"514d71c8-fd4e-4908-89a4-c0861e81ca8a",
|
||||
"35465081-a518-4321-8384-477ef5b29b8d",
|
||||
"de82a6f3-1a55-4996-ae68-35032678ff66"
|
||||
]
|
||||
}
|
||||
}
|
||||
24
third_party/EgoSpeak/LSTR/run_LSTR_inference.sh
vendored
Normal file
24
third_party/EgoSpeak/LSTR/run_LSTR_inference.sh
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
PATH_TO_CONFIG_FILE=configs/EasyCom/LSTR/lstr_aa_easycom.yaml
|
||||
PATH_TO_CHECKPOINT_FILE=checkpoints/lstr_easycom_AV.pth
|
||||
|
||||
echo "---"
|
||||
echo "CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES"
|
||||
echo "Config file: $PATH_TO_CONFIG_FILE"
|
||||
# echo "Checkpoint opt: $CHECKPOINT_OPT"
|
||||
# echo "Epoch opt: $EPOCH_OPT"
|
||||
|
||||
echo python tools/test_net.py \
|
||||
--config_file $PATH_TO_CONFIG_FILE \
|
||||
--gpu $CUDA_VISIBLE_DEVICES \
|
||||
MODEL.CHECKPOINT $PATH_TO_CHECKPOINT_FILE \
|
||||
MODEL.LSTR.INFERENCE_MODE batch \
|
||||
$@
|
||||
|
||||
python tools/test_net.py \
|
||||
--config_file $PATH_TO_CONFIG_FILE \
|
||||
--gpu $CUDA_VISIBLE_DEVICES \
|
||||
MODEL.CHECKPOINT $PATH_TO_CHECKPOINT_FILE \
|
||||
MODEL.LSTR.INFERENCE_MODE batch \
|
||||
$@
|
||||
25
third_party/EgoSpeak/LSTR/run_LSTR_train.sh
vendored
Normal file
25
third_party/EgoSpeak/LSTR/run_LSTR_train.sh
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
|
||||
PATH_TO_CONFIG_FILE=configs/EasyCom/LSTR/lstr_aa_easycom.yaml
|
||||
shift # Remove the first argument so $@ contains all other arguments
|
||||
|
||||
echo "Config file: $PATH_TO_CONFIG_FILE"
|
||||
echo "Additional options: $@"
|
||||
# EXAMPLE:
|
||||
# USE_WANDB True
|
||||
# INPUT.MODALITY "visual+motion+object"
|
||||
|
||||
echo "---"
|
||||
echo "CUDA_VISIBLE_DEVICES: $CUDA_VISIBLE_DEVICES"
|
||||
echo "Config file: $PATH_TO_CONFIG_FILE"
|
||||
|
||||
echo python tools/train_net.py \
|
||||
--config_file $PATH_TO_CONFIG_FILE \
|
||||
--gpu $CUDA_VISIBLE_DEVICES \
|
||||
$@
|
||||
|
||||
python tools/train_net.py \
|
||||
--config_file $PATH_TO_CONFIG_FILE \
|
||||
--gpu $CUDA_VISIBLE_DEVICES \
|
||||
$@
|
||||
|
||||
8
third_party/EgoSpeak/LSTR/setup.py
vendored
Normal file
8
third_party/EgoSpeak/LSTR/setup.py
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
setup(
|
||||
name='LSTR',
|
||||
version='0.1',
|
||||
packages=find_packages('src') + ['configs'],
|
||||
package_dir={'': 'src', 'configs': 'configs'},
|
||||
)
|
||||
2
third_party/EgoSpeak/LSTR/src/__init__.py
vendored
Normal file
2
third_party/EgoSpeak/LSTR/src/__init__.py
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
2
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/__init__.py
vendored
Normal file
2
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/__init__.py
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
4
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/config/__init__.py
vendored
Normal file
4
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/config/__init__.py
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from .defaults import get_cfg
|
||||
171
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/config/defaults.py
vendored
Normal file
171
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/config/defaults.py
vendored
Normal file
@@ -0,0 +1,171 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from yacs.config import CfgNode as CN
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Config Definition
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C = CN()
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Metadata
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.SEED = 0
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Model
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.MODEL = CN()
|
||||
_C.MODEL.MODEL_NAME = ''
|
||||
_C.MODEL.CHECKPOINT = ''
|
||||
_C.MODEL.EXCEPT_CLASSIFIER = False
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Feature Head
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.MODEL.FEATURE_HEAD = CN()
|
||||
_C.MODEL.FEATURE_HEAD.LINEAR_ENABLED = True
|
||||
_C.MODEL.FEATURE_HEAD.LINEAR_OUT_FEATURES = 1024
|
||||
_C.MODEL.FEATURE_HEAD.DROPOUT = 0.
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Transformer Network
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.MODEL.LSTR = CN()
|
||||
# Hyperparameters
|
||||
_C.MODEL.LSTR.NUM_HEADS = 8
|
||||
_C.MODEL.LSTR.DIM_FEEDFORWARD = 1024
|
||||
_C.MODEL.LSTR.DROPOUT = 0.2
|
||||
_C.MODEL.LSTR.DROPOUT_CLS = 0.0
|
||||
_C.MODEL.LSTR.ACTIVATION = 'relu'
|
||||
# Memory choices
|
||||
_C.MODEL.LSTR.AGES_MEMORY_SECONDS = 0
|
||||
_C.MODEL.LSTR.AGES_MEMORY_SAMPLE_RATE = 1
|
||||
_C.MODEL.LSTR.LONG_MEMORY_SECONDS = 0
|
||||
_C.MODEL.LSTR.LONG_MEMORY_SAMPLE_RATE = 1
|
||||
_C.MODEL.LSTR.WORK_MEMORY_SECONDS = 8
|
||||
_C.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE = 1
|
||||
_C.MODEL.LSTR.WORK_MEMORY_USE_PE = True
|
||||
_C.MODEL.LSTR.LONG_MEMORY_USE_PE = False
|
||||
_C.MODEL.LSTR.LONG_MEMORY_INCLUDE_WORK = False
|
||||
_C.MODEL.LSTR.LONG_MEMORY_INCLUDE_WORK2 = False
|
||||
# Anticipation choices
|
||||
_C.MODEL.LSTR.ANTICIPATION_SECONDS = 0
|
||||
_C.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE = 1
|
||||
# for EK55/100
|
||||
_C.MODEL.LSTR.V_N_CLASSIFIER = False
|
||||
_C.MODEL.LSTR.LOSS_ANTICIPATE_ONLY = False
|
||||
_C.MODEL.LSTR.LOSS_WEIGHT_V = 1.
|
||||
_C.MODEL.LSTR.LOSS_WEIGHT_N = 1.
|
||||
_C.MODEL.LSTR.FUSION_WEIGHT_V = 1.
|
||||
_C.MODEL.LSTR.FUSION_WEIGHT_N = 1.
|
||||
# Design choices
|
||||
_C.MODEL.LSTR.ENC_ATTENTION_TYPE = 'dotproduct'
|
||||
_C.MODEL.LSTR.ENC_ATTENTION_DECAY = 1.0
|
||||
_C.MODEL.LSTR.ENC_MODULE = [
|
||||
[16, 1, True], [32, 2, True]
|
||||
]
|
||||
_C.MODEL.LSTR.DEC_MODULE = [-1, 2, True]
|
||||
_C.MODEL.LSTR.FC_NORM = False
|
||||
# Inference modes
|
||||
_C.MODEL.LSTR.INFERENCE_MODE = 'batch'
|
||||
# Position Embedding
|
||||
# _C.MODEL.LSTR.POSITION_EMBEDDING = 'default'
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Criterion
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.MODEL.CRITERIONS = [['MCE', {}]]
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Data
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.DATA = CN()
|
||||
_C.DATA.DATA_INFO = 'data/data_info.json'
|
||||
_C.DATA.DATA_NAME = None
|
||||
_C.DATA.DATA_ROOT = None
|
||||
_C.DATA.CLASS_NAMES = None
|
||||
_C.DATA.NUM_CLASSES = None
|
||||
_C.DATA.IGNORE_INDEX = None
|
||||
_C.DATA.METRICS = None
|
||||
_C.DATA.FPS = None
|
||||
_C.DATA.TRAIN_SESSION_SET = None
|
||||
_C.DATA.TEST_SESSION_SET = None
|
||||
_C.DATA.EK_EXT_PATH = 'external/rulstm/RULSTM/data/ek55/'
|
||||
_C.DATA.CLIP_MIXUP_RATE = 0.
|
||||
_C.DATA.CLIP_MIXUP_SAMPLE = 'uniform'
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Input
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.INPUT = CN()
|
||||
_C.INPUT.MODALITY = 'twostream'
|
||||
# 'twostream'='visual+motion', 'threestream'='visual+motion+object'
|
||||
_C.INPUT.VISUAL_FEATURE = 'rgb_anet_resnet50'
|
||||
_C.INPUT.MOTION_FEATURE = 'flow_anet_resnet50'
|
||||
_C.INPUT.OBJECT_FEATURE = 'flow_anet_resnet50'
|
||||
_C.INPUT.TARGET_PERFRAME = 'target_perframe'
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Data Loader
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.DATA_LOADER = CN()
|
||||
_C.DATA_LOADER.BATCH_SIZE = 32
|
||||
_C.DATA_LOADER.NUM_WORKERS = 4
|
||||
_C.DATA_LOADER.PIN_MEMORY = False
|
||||
|
||||
# _C.MODEL.MINIROAD.NUM_WORKERS = 4
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Solver
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.SOLVER = CN()
|
||||
_C.SOLVER.START_EPOCH = 1
|
||||
_C.SOLVER.NUM_EPOCHS = 50
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Optimizer
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.SOLVER.OPTIMIZER = 'adam'
|
||||
_C.SOLVER.BASE_LR = 0.00005
|
||||
_C.SOLVER.WEIGHT_DECAY = 0.00005
|
||||
_C.SOLVER.MOMENTUM = 0.9
|
||||
|
||||
# _C.MODEL.MINIROAD.LR = 0.0001
|
||||
# _C.MODEL.MINIROAD.WEIGHT_DECAY = 0.05
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Scheduler
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.SOLVER.SCHEDULER = CN()
|
||||
_C.SOLVER.SCHEDULER.SCHEDULER_NAME = None
|
||||
_C.SOLVER.SCHEDULER.MILESTONES = []
|
||||
_C.SOLVER.SCHEDULER.GAMMA = 0.1
|
||||
_C.SOLVER.SCHEDULER.WARMUP_FACTOR = 0.3
|
||||
_C.SOLVER.SCHEDULER.WARMUP_EPOCHS = 10.0
|
||||
_C.SOLVER.SCHEDULER.WARMUP_METHOD = 'linear'
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Others
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.SOLVER.PHASES = ['train', 'test']
|
||||
_C.SOLVER.SAVE_EVERY_EPOCHS = 10
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.OUTPUT_DIR = 'checkpoints'
|
||||
_C.SESSION = ''
|
||||
_C.USE_WANDB = False
|
||||
_C.SAVE_HPARAM = False
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Misc
|
||||
# ---------------------------------------------------------------------------- #
|
||||
_C.VERBOSE = False
|
||||
|
||||
|
||||
def get_cfg():
|
||||
return _C.clone()
|
||||
20
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/config/feature_sizes.py
vendored
Normal file
20
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/config/feature_sizes.py
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
# dimension of features extracted from different models
|
||||
# used in feature_head, dataset, models
|
||||
|
||||
FEATURE_SIZES = {
|
||||
'rgb_anet_resnet50': 2048,
|
||||
'flow_anet_resnet50': 2048,
|
||||
'rgb_kinetics_bninception': 1024,
|
||||
'flow_kinetics_bninception': 1024,
|
||||
'rgb_farl': 512,
|
||||
'rgb_kinetics_resnet50': 2048,
|
||||
'rgb_kinetics_resnet50_debug': 2048,
|
||||
'rgb_kinetics_resnet50_debug_resized': 2048,
|
||||
'flow_kinetics_resnet50': 2048,
|
||||
'flow_nv_kinetics_bninception': 1024,
|
||||
'rgb_kinetics_i3d': 2048,
|
||||
'flow_kinetics_i3d': 2048,
|
||||
'audio_wav2vec2': 5120,
|
||||
'audio_wav2vec2_left': 5120,
|
||||
'audio_wav2vec2_right': 5120,
|
||||
}
|
||||
4
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/criterions/__init__.py
vendored
Normal file
4
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/criterions/__init__.py
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from .criterions import build_criterion
|
||||
356
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/criterions/criterions.py
vendored
Normal file
356
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/criterions/criterions.py
vendored
Normal file
@@ -0,0 +1,356 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
__all__ = ['build_criterion']
|
||||
|
||||
import os.path as osp
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from rekognition_online_action_detection.utils.registry import Registry
|
||||
|
||||
CRITERIONS = Registry()
|
||||
|
||||
|
||||
@CRITERIONS.register('BCE')
|
||||
class BinaryCrossEntropyLoss(nn.Module):
|
||||
|
||||
def __init__(self, reduction='mean', ignore_index=-100):
|
||||
super(BinaryCrossEntropyLoss, self).__init__()
|
||||
|
||||
self.criterion = nn.BCEWithLogitsLoss(reduction=reduction)
|
||||
|
||||
def forward(self, input, target):
|
||||
return self.criterion(input, target)
|
||||
|
||||
|
||||
@CRITERIONS.register('SCE')
|
||||
class SingleCrossEntropyLoss(nn.Module):
|
||||
|
||||
def __init__(self, reduction='mean', ignore_index=-100):
|
||||
super(SingleCrossEntropyLoss, self).__init__()
|
||||
|
||||
self.criterion = nn.CrossEntropyLoss(
|
||||
reduction=reduction, ignore_index=ignore_index)
|
||||
|
||||
def forward(self, input, target):
|
||||
return self.criterion(input, target)
|
||||
|
||||
|
||||
@CRITERIONS.register('MCE')
|
||||
class MultipCrossEntropyLoss(nn.Module):
|
||||
|
||||
def __init__(self, reduction='mean', ignore_index=-100):
|
||||
super(MultipCrossEntropyLoss, self).__init__()
|
||||
|
||||
self.reduction = reduction
|
||||
self.ignore_index = ignore_index
|
||||
|
||||
def forward(self, input, target):
|
||||
logsoftmax = nn.LogSoftmax(dim=-1).to(input.device)
|
||||
|
||||
# LSTR transform input from [16,50,3] to [800,3]
|
||||
output = torch.sum(-target * logsoftmax(input), dim=1)
|
||||
|
||||
if self.reduction == 'mean':
|
||||
return torch.mean(output)
|
||||
elif self.reduction == 'sum':
|
||||
return torch.sum(output)
|
||||
else:
|
||||
return output
|
||||
|
||||
# if self.ignore_index >= 0:
|
||||
# notice_index = [i for i in range(target.shape[-1]) if i != self.ignore_index]
|
||||
# output = torch.sum(-target[:, notice_index] * logsoftmax(input[:, notice_index]), dim=1)
|
||||
# if (target[:, self.ignore_index] != 1).sum().item() == 0:
|
||||
# return torch.mean(torch.zeros_like(output))
|
||||
# if self.reduction == 'mean':
|
||||
# return torch.mean(output[target[:, self.ignore_index] != 1])
|
||||
# elif self.reduction == 'sum':
|
||||
# return torch.sum(output[target[:, self.ignore_index] != 1])
|
||||
# else:
|
||||
# return output[target[:, self.ignore_index] != 1]
|
||||
# else:
|
||||
# output = torch.sum(-target * logsoftmax(input), dim=1)
|
||||
|
||||
# if self.reduction == 'mean':
|
||||
# return torch.mean(output)
|
||||
# elif self.reduction == 'sum':
|
||||
# return torch.sum(output)
|
||||
# else:
|
||||
# return output
|
||||
|
||||
@CRITERIONS.register('MCE_MiniROAD')
|
||||
class MiniROADMultipCrossEntropyLoss(nn.Module):
|
||||
|
||||
def __init__(self, reduction='mean', ignore_index=-100):
|
||||
super(MiniROADMultipCrossEntropyLoss, self).__init__()
|
||||
|
||||
self.reduction = reduction
|
||||
self.ignore_index = ignore_index
|
||||
|
||||
def forward(self, input, target):
|
||||
logsoftmax = nn.LogSoftmax(dim=-1).to(input.device)
|
||||
output = torch.sum(-F.normalize(target) * logsoftmax(input), dim=1) # B
|
||||
if self.reduction == 'mean':
|
||||
loss = torch.mean(output)
|
||||
elif self.reduction == 'sum':
|
||||
loss = torch.sum(output)
|
||||
|
||||
return loss
|
||||
|
||||
@CRITERIONS.register('MCE_ANT')
|
||||
class AntMultipCrossEntropyLoss(nn.Module):
|
||||
|
||||
def __init__(self, reduction='mean', ignore_index=-100):
|
||||
super(AntMultipCrossEntropyLoss, self).__init__()
|
||||
|
||||
self.reduction = reduction
|
||||
self.ignore_index = ignore_index
|
||||
|
||||
# only ant_target loss
|
||||
def forward(self, input, target):
|
||||
logsoftmax = nn.LogSoftmax(dim=-1).to(input.device)
|
||||
import pdb; pdb.set_trace()
|
||||
if self.ignore_index >= 0:
|
||||
notice_index = [i for i in range(target.shape[-1]) if i != self.ignore_index]
|
||||
import pdb; pdb.set_trace()
|
||||
# input dim: [16, 128, 10, 3]
|
||||
# target.shape: [16, 10, 3]
|
||||
output = torch.sum()
|
||||
|
||||
output = torch.sum(-target[:, notice_index] * logsoftmax(input[:, notice_index]), dim=1)
|
||||
if (target[:, self.ignore_index] != 1).sum().item() == 0:
|
||||
return torch.mean(torch.zeros_like(output))
|
||||
if self.reduction == 'mean':
|
||||
return torch.mean(output[target[:, self.ignore_index] != 1])
|
||||
elif self.reduction == 'sum':
|
||||
return torch.sum(output[target[:, self.ignore_index] != 1])
|
||||
else:
|
||||
return output[target[:, self.ignore_index] != 1]
|
||||
else:
|
||||
output = torch.sum(-target * logsoftmax(input), dim=1)
|
||||
|
||||
if self.reduction == 'mean':
|
||||
return torch.mean(output)
|
||||
elif self.reduction == 'sum':
|
||||
return torch.sum(output)
|
||||
else:
|
||||
return output
|
||||
|
||||
|
||||
@CRITERIONS.register('MCE_EQL')
|
||||
class MultipCrossEntropyEqualizedLoss(nn.Module):
|
||||
|
||||
def __init__(self, gamma=0.95, lambda_=3e-3, reduction='mean', ignore_index=-100,
|
||||
anno_path='external/rulstm/RULSTM/data/ek55/'):
|
||||
super(MultipCrossEntropyEqualizedLoss, self).__init__()
|
||||
|
||||
# get label distribution
|
||||
segment_list = pd.read_csv(osp.join(anno_path, 'training.csv'),
|
||||
names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
||||
skipinitialspace=True)
|
||||
freq_info = np.zeros((max(segment_list['action']) + 1,))
|
||||
assert ignore_index == 0
|
||||
for segment in segment_list.iterrows():
|
||||
freq_info[segment[1]['action']] += 1.
|
||||
freq_info = freq_info / freq_info.sum()
|
||||
self.freq_info = torch.FloatTensor(freq_info)
|
||||
|
||||
self.gamma = gamma
|
||||
self.lambda_ = lambda_
|
||||
self.reduction = reduction
|
||||
self.ignore_index = ignore_index
|
||||
|
||||
def forward(self, input, target):
|
||||
logsoftmax = nn.LogSoftmax(dim=1).to(input.device)
|
||||
|
||||
|
||||
bg_target = target[:, self.ignore_index]
|
||||
notice_index = [i for i in range(target.shape[-1]) if i != self.ignore_index]
|
||||
input = input[:, notice_index]
|
||||
target = target[:, notice_index]
|
||||
|
||||
weight = input.new_zeros(len(notice_index))
|
||||
weight[self.freq_info < self.lambda_] = 1.
|
||||
weight = weight.view(1, -1).repeat(input.shape[0], 1)
|
||||
|
||||
eql_w = 1 - (torch.rand_like(target) < self.gamma) * weight * (1 - target)
|
||||
input = torch.log(eql_w + 1e-8) + input
|
||||
|
||||
output = torch.sum(-target * logsoftmax(input), dim=1)
|
||||
if (bg_target != 1).sum().item() == 0:
|
||||
return torch.mean(torch.zeros_like(output))
|
||||
if self.reduction == 'mean':
|
||||
return torch.mean(output[bg_target != 1])
|
||||
elif self.reduction == 'sum':
|
||||
return torch.sum(output[bg_target != 1])
|
||||
else:
|
||||
return output[bg_target != 1]
|
||||
|
||||
|
||||
@CRITERIONS.register('EQLv2')
|
||||
class EQLv2Loss(nn.Module):
|
||||
|
||||
def __init__(self, gamma=12, mu=0.8, alpha=4.0, reduction='mean', ignore_index=-100,
|
||||
num_classes=3806):
|
||||
super(EQLv2Loss, self).__init__()
|
||||
self.num_classes = num_classes
|
||||
|
||||
self.gamma = gamma
|
||||
self.mu = mu
|
||||
self.alpha = alpha
|
||||
self.ignore_index = ignore_index
|
||||
|
||||
self._pos_grad = None
|
||||
self._neg_grad = None
|
||||
self.pos_neg = None
|
||||
|
||||
def _func(x, gamma, mu):
|
||||
return 1 / (1 + torch.exp(-gamma * (x - mu)))
|
||||
self.map_func = partial(_func, gamma=self.gamma, mu=self.mu)
|
||||
|
||||
def forward(self, input, target):
|
||||
self.n_i, self.n_c = input.size()
|
||||
|
||||
pos_w, neg_w = self.get_weight(input)
|
||||
|
||||
weight = pos_w * target + neg_w * (1 - target)
|
||||
|
||||
cls_loss = F.binary_cross_entropy_with_logits(input, target, reduction='none')
|
||||
cls_loss = torch.sum(cls_loss * weight) / self.n_i
|
||||
|
||||
self.collect_grad(input.detach(), target.detach(), weight.detach())
|
||||
|
||||
return cls_loss
|
||||
|
||||
def collect_grad(self, cls_score, target, weight):
|
||||
prob = torch.sigmoid(cls_score)
|
||||
grad = target * (prob - 1) + (1 - target) * prob
|
||||
grad = torch.abs(grad)
|
||||
|
||||
# do not collect grad for objectiveness branch [:-1]
|
||||
pos_grad = torch.sum(grad * target * weight, dim=0)[1:]
|
||||
neg_grad = torch.sum(grad * (1 - target) * weight, dim=0)[1:]
|
||||
|
||||
# dist.all_reduce(pos_grad)
|
||||
# dist.all_reduce(neg_grad)
|
||||
|
||||
self._pos_grad += pos_grad
|
||||
self._neg_grad += neg_grad
|
||||
self.pos_neg = self._pos_grad / (self._neg_grad + 1e-10)
|
||||
|
||||
def get_weight(self, cls_score):
|
||||
if self._pos_grad is None:
|
||||
self._pos_grad = cls_score.new_zeros(self.num_classes)
|
||||
self._neg_grad = cls_score.new_zeros(self.num_classes)
|
||||
neg_w = cls_score.new_zeros((self.n_i, self.n_c))
|
||||
pos_w = cls_score.new_zeros((self.n_i, self.n_c))
|
||||
else:
|
||||
neg_w = torch.cat([cls_score.new_ones(1), self.map_func(self.pos_neg)])
|
||||
pos_w = 1 + self.alpha * (1 - neg_w)
|
||||
neg_w = neg_w.view(1, -1).expand(self.n_i, self.n_c)
|
||||
pos_w = pos_w.view(1, -1).expand(self.n_i, self.n_c)
|
||||
return pos_w, neg_w
|
||||
|
||||
|
||||
@CRITERIONS.register('MCE_SEESAW')
|
||||
class MultipCrossEntropySeesawLoss(nn.Module):
|
||||
|
||||
def __init__(self, reduction='mean', ignore_index=-100,
|
||||
p=0.8, q=2.0, eps=1e-2,
|
||||
anno_path='external/rulstm/RULSTM/data/ek55/'):
|
||||
super(MultipCrossEntropySeesawLoss, self).__init__()
|
||||
|
||||
# get label distribution
|
||||
segment_list = pd.read_csv(osp.join(anno_path, 'training.csv'),
|
||||
names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
||||
skipinitialspace=True)
|
||||
self.num_classes = max(segment_list['action']) + 1
|
||||
self.cum_samples = torch.zeros(self.num_classes, dtype=torch.float)
|
||||
|
||||
self.p = p
|
||||
self.q = q
|
||||
self.eps = eps
|
||||
|
||||
self.reduction = reduction
|
||||
assert ignore_index == 0
|
||||
self.ignore_index = ignore_index
|
||||
|
||||
def forward(self, input, target):
|
||||
logsoftmax = nn.LogSoftmax(dim=1).to(input.device)
|
||||
self.cum_samples = self.cum_samples.to(input.device)
|
||||
|
||||
bg_target = target[:, self.ignore_index]
|
||||
notice_index = [i for i in range(target.shape[-1]) if i != self.ignore_index]
|
||||
input = input[:, notice_index]
|
||||
target = target[:, notice_index]
|
||||
|
||||
# accumulate the samples for each category
|
||||
self.cum_samples += target.sum(0)
|
||||
|
||||
seesaw_weights = input.new_ones(target.size())
|
||||
|
||||
if self.p > 0:
|
||||
sample_ratio_matrix = self.cum_samples[None, :].clamp(min=1) / self.cum_samples[:, None].clamp(min=1)
|
||||
index = (sample_ratio_matrix < 1.0).float()
|
||||
sample_weights = sample_ratio_matrix.pow(self.p) * index + (1 - index)
|
||||
for ni in range(input.shape[0]):
|
||||
true_target = target[ni, :].nonzero(as_tuple=True)[0]
|
||||
## TODO: study mean/min/max/sum
|
||||
if len(true_target) > 0:
|
||||
mitigation_factor = torch.min(sample_weights[true_target, :], dim=0).values
|
||||
seesaw_weights[ni, :] = seesaw_weights[ni, :] * mitigation_factor
|
||||
if self.q > 0:
|
||||
scores = F.softmax(input.detach(), dim=1)
|
||||
self_scores = input.new_ones([input.shape[0]])
|
||||
for ni in range(input.shape[0]):
|
||||
true_target = target[ni, :].nonzero(as_tuple=True)[0]
|
||||
## TODO: study mean/min/max/sum
|
||||
if len(true_target) > 0:
|
||||
self_scores[ni] = torch.min(scores[ni, true_target])
|
||||
score_matrix = scores / self_scores[:, None].clamp(min=self.eps)
|
||||
index = (score_matrix > 1.0).float()
|
||||
compensation_factor = score_matrix.pow(self.q) * index + (1 - index)
|
||||
seesaw_weights = seesaw_weights * compensation_factor
|
||||
|
||||
input = input + (seesaw_weights.log() * (1 - target))
|
||||
|
||||
output = torch.sum(-target * logsoftmax(input), dim=1)
|
||||
if (bg_target != 1).sum().item() == 0:
|
||||
return torch.mean(torch.zeros_like(output))
|
||||
if self.reduction == 'mean':
|
||||
return torch.mean(output[bg_target != 1])
|
||||
elif self.reduction == 'sum':
|
||||
return torch.sum(output[bg_target != 1])
|
||||
else:
|
||||
return output[bg_target != 1]
|
||||
|
||||
|
||||
@CRITERIONS.register('PRED_FUTURE')
|
||||
class PredictFutureLoss(nn.Module):
|
||||
|
||||
def __init__(self, reduction='mean', ignore_index=-100):
|
||||
super(PredictFutureLoss, self).__init__()
|
||||
|
||||
self.reduction = reduction
|
||||
self.criterion = nn.MSELoss(reduction=reduction)
|
||||
|
||||
def forward(self, pred, true):
|
||||
return self.criterion(pred, true)
|
||||
|
||||
|
||||
def build_criterion(cfg, device=None):
|
||||
criterion = {}
|
||||
for name, params in cfg.MODEL.CRITERIONS:
|
||||
if name in CRITERIONS:
|
||||
if 'ignore_index' not in params:
|
||||
params['ignore_index'] = cfg.DATA.IGNORE_INDEX
|
||||
criterion[name] = CRITERIONS[name](**params).to(device)
|
||||
else:
|
||||
raise RuntimeError('Unknown criterion: {}'.format(name))
|
||||
return criterion
|
||||
5
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/datasets/__init__.py
vendored
Normal file
5
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/datasets/__init__.py
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from .datasets import build_data_loader, build_dataset
|
||||
from .perframe_data_layers import *
|
||||
31
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/datasets/datasets.py
vendored
Normal file
31
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/datasets/datasets.py
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
|
||||
|
||||
__all__ = [
|
||||
'build_dataset',
|
||||
'build_data_loader',
|
||||
]
|
||||
|
||||
import torch.utils.data as data
|
||||
from rekognition_online_action_detection.utils.registry import Registry
|
||||
|
||||
DATA_LAYERS = Registry()
|
||||
|
||||
|
||||
def build_dataset(cfg, phase, tag=''):
|
||||
data_names = ['YTConv', 'EGO4D']
|
||||
data_name = next((name for name in data_names if name in cfg.DATA.DATA_NAME), cfg.DATA.DATA_NAME)
|
||||
|
||||
data_layer = DATA_LAYERS[cfg.MODEL.MODEL_NAME + tag + data_name]
|
||||
return data_layer(cfg, phase)
|
||||
|
||||
|
||||
def build_data_loader(cfg, phase):
|
||||
data_loader = data.DataLoader(
|
||||
dataset=build_dataset(cfg, phase),
|
||||
batch_size=cfg.DATA_LOADER.BATCH_SIZE,
|
||||
shuffle=True if phase == 'train' else False,
|
||||
num_workers=cfg.DATA_LOADER.NUM_WORKERS,
|
||||
pin_memory=cfg.DATA_LOADER.PIN_MEMORY,
|
||||
)
|
||||
return data_loader
|
||||
453
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/datasets/perframe_data_layers.py
vendored
Normal file
453
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/datasets/perframe_data_layers.py
vendored
Normal file
@@ -0,0 +1,453 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
|
||||
import os.path as osp
|
||||
import warnings
|
||||
from bisect import bisect_right
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
import torch.utils.data as data
|
||||
from rekognition_online_action_detection.utils.ek_utils import (
|
||||
action_to_noun_map, action_to_verb_map)
|
||||
|
||||
from .datasets import DATA_LAYERS as registry
|
||||
|
||||
|
||||
@registry.register('LSTRTHUMOS')
|
||||
@registry.register('LSTRTVSeries')
|
||||
@registry.register('LSTREK55')
|
||||
@registry.register('LSTREK100')
|
||||
@registry.register('LSTRYTConv')
|
||||
@registry.register('LSTREGO4D')
|
||||
@registry.register('LSTREasyCom')
|
||||
class LSTRDataLayer(data.Dataset):
|
||||
|
||||
def __init__(self, cfg, phase='train'):
|
||||
self.cfg = cfg
|
||||
self.data_name = cfg.DATA.DATA_NAME
|
||||
self.data_root = cfg.DATA.DATA_ROOT
|
||||
self.clip_mixup_rate = cfg.DATA.CLIP_MIXUP_RATE
|
||||
self.clip_mixup_sample = cfg.DATA.CLIP_MIXUP_SAMPLE
|
||||
self.visual_feature = cfg.INPUT.VISUAL_FEATURE
|
||||
self.motion_feature = cfg.INPUT.MOTION_FEATURE
|
||||
self.object_feature = cfg.INPUT.OBJECT_FEATURE
|
||||
self.target_perframe = cfg.INPUT.TARGET_PERFRAME
|
||||
self.sessions = getattr(cfg.DATA, phase.upper() + '_SESSION_SET')
|
||||
self.long_memory_length = cfg.MODEL.LSTR.LONG_MEMORY_LENGTH
|
||||
self.long_memory_sample_rate = cfg.MODEL.LSTR.LONG_MEMORY_SAMPLE_RATE
|
||||
self.long_memory_num_samples = cfg.MODEL.LSTR.LONG_MEMORY_NUM_SAMPLES
|
||||
self.work_memory_length = cfg.MODEL.LSTR.WORK_MEMORY_LENGTH
|
||||
self.work_memory_sample_rate = cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE
|
||||
self.work_memory_num_samples = cfg.MODEL.LSTR.WORK_MEMORY_NUM_SAMPLES
|
||||
self.anticipation_length = cfg.MODEL.LSTR.ANTICIPATION_LENGTH
|
||||
self.anticipation_sample_rate = cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE
|
||||
self.anticipation_num_samples = cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES
|
||||
self.training = phase == 'train'
|
||||
self._init_dataset()
|
||||
|
||||
def shuffle(self):
|
||||
self._init_dataset()
|
||||
|
||||
def _init_dataset(self):
|
||||
self.inputs = []
|
||||
|
||||
if self.data_name.startswith('EK'):
|
||||
if self.data_name == 'EK55':
|
||||
path_to_data = 'external/rulstm/RULSTM/data/ek55/'
|
||||
elif self.data_name == 'EK100':
|
||||
path_to_data = 'external/rulstm/RULSTM/data/ek100/'
|
||||
else:
|
||||
raise ValueError
|
||||
segment_list = pd.read_csv(osp.join(path_to_data,
|
||||
'training.csv' if self.training else 'validation.csv'),
|
||||
names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
||||
skipinitialspace=True)
|
||||
segment_list['len_f'] = segment_list['end_f'] - segment_list['start_f']
|
||||
|
||||
self.segment_by_cls = {cls: segment_list[segment_list['action'] == cls]
|
||||
for cls in range(0, self.cfg.DATA.NUM_CLASSES - 1)}
|
||||
for session in self.sessions:
|
||||
target = np.load(osp.join(self.data_root, self.target_perframe, session + '.npy'))
|
||||
if self.cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
verb_target = np.load(osp.join(self.data_root, self.target_perframe.replace('target', 'verb'), session + '.npy'))
|
||||
noun_target = np.load(osp.join(self.data_root, self.target_perframe.replace('target', 'noun'), session + '.npy'))
|
||||
else:
|
||||
verb_target = target
|
||||
noun_target = target
|
||||
segments_per_session = segment_list[segment_list['video'] == session]
|
||||
for segment in segments_per_session.iterrows():
|
||||
start_tick = int(segment[1]['start_f'] / 30 * self.cfg.DATA.FPS)
|
||||
end_tick = int(segment[1]['end_f'] / 30 * self.cfg.DATA.FPS)
|
||||
start_tick += np.random.randint(self.anticipation_length) if self.training else 0
|
||||
start_tick = min(start_tick, end_tick)
|
||||
work_end = start_tick - self.anticipation_length
|
||||
work_start = work_end - self.work_memory_length
|
||||
segments_before_current = segments_per_session[segments_per_session['end_f'] < segment[1]['start_f']]
|
||||
if work_start < 0:
|
||||
continue
|
||||
self.inputs.append([
|
||||
session, work_start, work_end,
|
||||
target[work_start: work_end + self.anticipation_length],
|
||||
verb_target[work_start: work_end + self.anticipation_length],
|
||||
noun_target[work_start: work_end + self.anticipation_length],
|
||||
segments_before_current,
|
||||
])
|
||||
else:
|
||||
for session in self.sessions:
|
||||
target = np.load(osp.join(self.data_root, self.target_perframe, session + '.npy'))
|
||||
if self.cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
verb_target = np.load(osp.join(self.data_root, self.target_perframe.replace('target', 'verb'), session + '.npy'))
|
||||
noun_target = np.load(osp.join(self.data_root, self.target_perframe.replace('target', 'noun'), session + '.npy'))
|
||||
else:
|
||||
verb_target = target
|
||||
noun_target = target
|
||||
seed = np.random.randint(self.work_memory_length + self.anticipation_length) if self.training else 0
|
||||
for work_start, work_end in zip(
|
||||
range(seed, target.shape[0], self.work_memory_length + self.anticipation_length),
|
||||
range(seed + self.work_memory_length, target.shape[0] - self.anticipation_length, self.work_memory_length + self.anticipation_length)):
|
||||
self.inputs.append([
|
||||
session, work_start, work_end,
|
||||
target[work_start: work_end + self.anticipation_length],
|
||||
verb_target[work_start: work_end + self.anticipation_length],
|
||||
noun_target[work_start: work_end + self.anticipation_length],
|
||||
None,
|
||||
])
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def segment_sampler(self, start, end, num_samples):
|
||||
indices = np.linspace(start, end, num_samples)
|
||||
return np.sort(indices).astype(np.int32)
|
||||
|
||||
def uniform_sampler(self, start, end, num_samples, sample_rate):
|
||||
if start < 0:
|
||||
start = (end + 1) % sample_rate
|
||||
indices = np.arange(start, end + 1)[::sample_rate]
|
||||
padding = num_samples - indices.shape[0]
|
||||
if padding > 0:
|
||||
indices = np.concatenate((np.zeros(padding), indices))
|
||||
return np.sort(indices).astype(np.int32)
|
||||
|
||||
def __getitem__(self, index):
|
||||
(session, work_start, work_end, target,
|
||||
verb_target, noun_target, segments_before) = self.inputs[index]
|
||||
visual_inputs = np.load(
|
||||
osp.join(self.data_root, self.visual_feature, session + '.npy'), mmap_mode='r')
|
||||
motion_inputs = np.load(
|
||||
osp.join(self.data_root, self.motion_feature, session + '.npy'), mmap_mode='r')
|
||||
object_inputs = np.load(
|
||||
osp.join(self.data_root, self.object_feature, session + '.npy'), mmap_mode='r')
|
||||
|
||||
|
||||
target = np.concatenate((target[:self.work_memory_length:self.work_memory_sample_rate],
|
||||
target[self.work_memory_length::self.anticipation_sample_rate]),
|
||||
axis=0)
|
||||
if self.cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
verb_target = np.concatenate((verb_target[:self.work_memory_length:self.work_memory_sample_rate],
|
||||
verb_target[self.work_memory_length::self.anticipation_sample_rate]),
|
||||
axis=0)
|
||||
noun_target = np.concatenate((noun_target[:self.work_memory_length:self.work_memory_sample_rate],
|
||||
noun_target[self.work_memory_length::self.anticipation_sample_rate]),
|
||||
axis=0)
|
||||
|
||||
|
||||
work_indices = np.arange(work_start, work_end).clip(0)
|
||||
work_indices = work_indices[::self.work_memory_sample_rate]
|
||||
work_visual_inputs = visual_inputs[work_indices]
|
||||
work_motion_inputs = motion_inputs[work_indices]
|
||||
work_object_inputs = object_inputs[work_indices]
|
||||
|
||||
|
||||
if self.long_memory_num_samples > 0:
|
||||
|
||||
long_start, long_end = work_start - self.long_memory_length, work_start - 1
|
||||
if self.training:
|
||||
long_indices = self.segment_sampler(
|
||||
long_start,
|
||||
long_end,
|
||||
self.long_memory_num_samples).clip(0)
|
||||
else:
|
||||
long_indices = self.uniform_sampler(
|
||||
long_start,
|
||||
long_end,
|
||||
self.long_memory_num_samples,
|
||||
self.long_memory_sample_rate).clip(0)
|
||||
long_visual_inputs = visual_inputs[long_indices]
|
||||
long_motion_inputs = motion_inputs[long_indices]
|
||||
long_object_inputs = object_inputs[long_indices]
|
||||
|
||||
if self.training and self.clip_mixup_rate > 0:
|
||||
assert segments_before is not None
|
||||
segments_before = segments_before[segments_before['end_f'] / 30 * self.cfg.DATA.FPS > long_start]
|
||||
if self.clip_mixup_sample == 'uniform':
|
||||
prob = None
|
||||
elif self.clip_mixup_sample == 'by_length':
|
||||
prob = (segments_before['len_f']).tolist()
|
||||
prob = prob / np.sum(prob)
|
||||
else:
|
||||
raise ValueError
|
||||
num_clip = int(len(segments_before) * self.clip_mixup_rate)
|
||||
segments_to_mixup = segments_before.sample(num_clip, replace=False,
|
||||
weights=prob)
|
||||
for old_segment in segments_to_mixup.iterrows():
|
||||
old_start_tick = int(old_segment[1]['start_f'] / 30 * self.cfg.DATA.FPS)
|
||||
old_end_tick = int(np.ceil(old_segment[1]['end_f'] / 30 * self.cfg.DATA.FPS))
|
||||
old_action = old_segment[1]['action']
|
||||
old_vid = old_segment[1]['video']
|
||||
assert old_vid == session
|
||||
segment_same_class = self.segment_by_cls[old_action]
|
||||
segment_same_class = segment_same_class[segment_same_class['video'] != old_vid]
|
||||
valid_segments = segment_same_class[segment_same_class['len_f'] > old_segment[1]['len_f']]
|
||||
if len(valid_segments) == 0:
|
||||
continue
|
||||
else:
|
||||
sample_segment = valid_segments.sample(1)
|
||||
new_vid = sample_segment['video'].values[0]
|
||||
new_start_tick = int(sample_segment['start_f'] / 30 * self.cfg.DATA.FPS)
|
||||
new_end_tick = int(sample_segment['end_f'] / 30 * self.cfg.DATA.FPS)
|
||||
|
||||
new_visual_inputs = np.load(
|
||||
osp.join(self.data_root, self.visual_feature, new_vid + '.npy'), mmap_mode='r')
|
||||
new_motion_inputs = np.load(
|
||||
osp.join(self.data_root, self.motion_feature, new_vid + '.npy'), mmap_mode='r')
|
||||
new_object_inputs = np.load(
|
||||
osp.join(self.data_root, self.object_feature, new_vid + '.npy'), mmap_mode='r')
|
||||
|
||||
sel_indices = np.where((long_indices >= old_start_tick) & (long_indices <= old_end_tick))
|
||||
|
||||
shift = np.random.randint(new_end_tick - new_start_tick - len(sel_indices) + 1)
|
||||
long_visual_inputs[sel_indices] = new_visual_inputs[new_start_tick + shift:new_start_tick + shift + len(sel_indices)]
|
||||
long_motion_inputs[sel_indices] = new_motion_inputs[new_start_tick + shift:new_start_tick + shift + len(sel_indices)]
|
||||
long_object_inputs[sel_indices] = new_object_inputs[new_start_tick + shift:new_start_tick + shift + len(sel_indices)]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
memory_key_padding_mask = np.zeros(long_indices.shape[0])
|
||||
last_zero = bisect_right(long_indices, 0) - 1
|
||||
if last_zero > 0:
|
||||
memory_key_padding_mask[:last_zero] = float('-inf')
|
||||
else:
|
||||
long_visual_inputs = None
|
||||
long_motion_inputs = None
|
||||
long_object_inputs = None
|
||||
memory_key_padding_mask = None
|
||||
|
||||
|
||||
if (long_visual_inputs is not None and
|
||||
long_motion_inputs is not None and
|
||||
long_object_inputs is not None):
|
||||
fusion_visual_inputs = np.concatenate((long_visual_inputs, work_visual_inputs))
|
||||
fusion_motion_inputs = np.concatenate((long_motion_inputs, work_motion_inputs))
|
||||
fusion_object_inputs = np.concatenate((long_object_inputs, work_object_inputs))
|
||||
else:
|
||||
fusion_visual_inputs = work_visual_inputs
|
||||
fusion_motion_inputs = work_motion_inputs
|
||||
fusion_object_inputs = work_object_inputs
|
||||
|
||||
|
||||
fusion_visual_inputs = torch.as_tensor(fusion_visual_inputs.astype(np.float32))
|
||||
fusion_motion_inputs = torch.as_tensor(fusion_motion_inputs.astype(np.float32))
|
||||
fusion_object_inputs = torch.as_tensor(fusion_object_inputs.astype(np.float32))
|
||||
target = torch.as_tensor(target.astype(np.float32))
|
||||
if self.cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
verb_target = torch.as_tensor(verb_target.astype(np.float32))
|
||||
noun_target = torch.as_tensor(noun_target.astype(np.float32))
|
||||
target = (target, verb_target, noun_target)
|
||||
|
||||
assert fusion_visual_inputs.ndim == fusion_motion_inputs.ndim
|
||||
if fusion_visual_inputs.ndim == 4:
|
||||
|
||||
fusion_visual_inputs = fusion_visual_inputs.permute(0, 2, 3, 1)
|
||||
fusion_motion_inputs = fusion_motion_inputs.permute(0, 2, 3, 1)
|
||||
fusion_visual_inputs = fusion_visual_inputs.mean((1, 2))
|
||||
fusion_motion_inputs = fusion_motion_inputs.mean((1, 2))
|
||||
|
||||
|
||||
|
||||
|
||||
if memory_key_padding_mask is not None:
|
||||
memory_key_padding_mask = torch.as_tensor(memory_key_padding_mask.astype(np.float32))
|
||||
return (fusion_visual_inputs, fusion_motion_inputs,
|
||||
fusion_object_inputs, memory_key_padding_mask, target)
|
||||
else:
|
||||
return (fusion_visual_inputs, fusion_motion_inputs,
|
||||
fusion_object_inputs, fusion_object_inputs, target)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.inputs)
|
||||
|
||||
|
||||
@registry.register('LSTRBatchInferenceTHUMOS')
|
||||
@registry.register('LSTRBatchInferenceEGO4D')
|
||||
@registry.register('LSTRBatchInferenceEasyCom')
|
||||
@registry.register('LSTRBatchInferenceTVSeries')
|
||||
@registry.register('LSTRBatchInferenceEK55')
|
||||
@registry.register('LSTRBatchInferenceEK100')
|
||||
class LSTRBatchInferenceDataLayer(data.Dataset):
|
||||
|
||||
def __init__(self, cfg, phase='test'):
|
||||
self.cfg = cfg
|
||||
self.data_root = cfg.DATA.DATA_ROOT
|
||||
self.visual_feature = cfg.INPUT.VISUAL_FEATURE
|
||||
self.motion_feature = cfg.INPUT.MOTION_FEATURE
|
||||
self.object_feature = cfg.INPUT.OBJECT_FEATURE
|
||||
self.target_perframe = cfg.INPUT.TARGET_PERFRAME
|
||||
self.sessions = getattr(cfg.DATA, phase.upper() + '_SESSION_SET')
|
||||
self.long_memory_length = cfg.MODEL.LSTR.LONG_MEMORY_LENGTH
|
||||
self.long_memory_sample_rate = cfg.MODEL.LSTR.LONG_MEMORY_SAMPLE_RATE
|
||||
self.long_memory_num_samples = cfg.MODEL.LSTR.LONG_MEMORY_NUM_SAMPLES
|
||||
self.work_memory_length = cfg.MODEL.LSTR.WORK_MEMORY_LENGTH
|
||||
self.work_memory_sample_rate = cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE
|
||||
self.work_memory_num_samples = cfg.MODEL.LSTR.WORK_MEMORY_NUM_SAMPLES
|
||||
self.anticipation_length = cfg.MODEL.LSTR.ANTICIPATION_LENGTH
|
||||
self.anticipation_sample_rate = cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE
|
||||
self.anticipation_num_samples = cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES
|
||||
|
||||
if phase != 'test':
|
||||
warnings.warn('originally, phase must be `test` for batch inference, got {}'.format(phase))
|
||||
|
||||
|
||||
self.inputs = []
|
||||
for session in self.sessions:
|
||||
target = np.load(osp.join(self.data_root, self.target_perframe, session + '.npy'))
|
||||
if self.cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
verb_target = np.load(osp.join(self.data_root, self.target_perframe.replace('target', 'verb'), session + '.npy'))
|
||||
noun_target = np.load(osp.join(self.data_root, self.target_perframe.replace('target', 'noun'), session + '.npy'))
|
||||
else:
|
||||
verb_target = target
|
||||
noun_target = target
|
||||
for work_start, work_end in zip(
|
||||
range(0, target.shape[0] + 1),
|
||||
range(self.work_memory_length, target.shape[0] + 1)):
|
||||
self.inputs.append([
|
||||
session, work_start, work_end,
|
||||
target[work_start: work_end],
|
||||
verb_target[work_start: work_end],
|
||||
noun_target[work_start: work_end],
|
||||
target.shape[0]
|
||||
])
|
||||
|
||||
def uniform_sampler(self, start, end, num_samples, sample_rate):
|
||||
if start < 0:
|
||||
start = (end + 1) % sample_rate
|
||||
indices = np.arange(start, end + 1)[::sample_rate]
|
||||
padding = num_samples - indices.shape[0]
|
||||
if padding > 0:
|
||||
indices = np.concatenate((np.zeros(padding), indices))
|
||||
return np.sort(indices).astype(np.int32)
|
||||
|
||||
def __getitem__(self, index):
|
||||
session, work_start, work_end, target, verb_target, noun_target, num_frames = self.inputs[index]
|
||||
visual_inputs = np.load(
|
||||
osp.join(self.data_root, self.visual_feature, session + '.npy'), mmap_mode='r')
|
||||
motion_inputs = np.load(
|
||||
osp.join(self.data_root, self.motion_feature, session + '.npy'), mmap_mode='r')
|
||||
object_inputs = np.load(
|
||||
osp.join(self.data_root, self.object_feature, session + '.npy'), mmap_mode='r')
|
||||
|
||||
|
||||
target = target[::self.work_memory_sample_rate]
|
||||
verb_target = verb_target[::self.work_memory_sample_rate]
|
||||
noun_target = noun_target[::self.work_memory_sample_rate]
|
||||
|
||||
|
||||
work_indices = np.arange(work_start, work_end).clip(0)
|
||||
work_indices = work_indices[::self.work_memory_sample_rate]
|
||||
work_visual_inputs = visual_inputs[work_indices]
|
||||
work_motion_inputs = motion_inputs[work_indices]
|
||||
work_object_inputs = object_inputs[work_indices]
|
||||
|
||||
|
||||
if self.long_memory_num_samples > 0:
|
||||
|
||||
long_start, long_end = work_start - self.long_memory_length, work_start - 1
|
||||
long_indices = self.uniform_sampler(
|
||||
long_start,
|
||||
long_end,
|
||||
self.long_memory_num_samples,
|
||||
self.long_memory_sample_rate).clip(0)
|
||||
long_visual_inputs = visual_inputs[long_indices]
|
||||
long_motion_inputs = motion_inputs[long_indices]
|
||||
long_object_inputs = object_inputs[long_indices]
|
||||
|
||||
|
||||
memory_key_padding_mask = np.zeros(long_indices.shape[0])
|
||||
last_zero = bisect_right(long_indices, 0) - 1
|
||||
if last_zero > 0:
|
||||
memory_key_padding_mask[:last_zero] = float('-inf')
|
||||
else:
|
||||
long_visual_inputs = None
|
||||
long_motion_inputs = None
|
||||
long_object_inputs = None
|
||||
memory_key_padding_mask = None
|
||||
|
||||
|
||||
if long_visual_inputs is not None and long_motion_inputs is not None:
|
||||
fusion_visual_inputs = np.concatenate((long_visual_inputs, work_visual_inputs))
|
||||
fusion_motion_inputs = np.concatenate((long_motion_inputs, work_motion_inputs))
|
||||
fusion_object_inputs = np.concatenate((long_object_inputs, work_object_inputs))
|
||||
else:
|
||||
fusion_visual_inputs = work_visual_inputs
|
||||
fusion_motion_inputs = work_motion_inputs
|
||||
fusion_object_inputs = work_object_inputs
|
||||
|
||||
|
||||
fusion_visual_inputs = torch.as_tensor(fusion_visual_inputs.astype(np.float32))
|
||||
fusion_motion_inputs = torch.as_tensor(fusion_motion_inputs.astype(np.float32))
|
||||
fusion_object_inputs = torch.as_tensor(fusion_object_inputs.astype(np.float32))
|
||||
target = torch.as_tensor(target.astype(np.float32))
|
||||
|
||||
if self.cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
verb_target = torch.as_tensor(verb_target.astype(np.float32))
|
||||
noun_target = torch.as_tensor(noun_target.astype(np.float32))
|
||||
target = (target, verb_target, noun_target)
|
||||
|
||||
if memory_key_padding_mask is not None:
|
||||
memory_key_padding_mask = torch.as_tensor(memory_key_padding_mask.astype(np.float32))
|
||||
return (fusion_visual_inputs, fusion_motion_inputs,
|
||||
fusion_object_inputs, memory_key_padding_mask, target,
|
||||
session, work_indices, num_frames)
|
||||
else:
|
||||
return (fusion_visual_inputs, fusion_motion_inputs,
|
||||
fusion_object_inputs, target,
|
||||
session, work_indices, num_frames)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.inputs)
|
||||
6
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/__init__.py
vendored
Normal file
6
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/__init__.py
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from .engines import do_inference, do_train
|
||||
from .lstr.lstr_inference import *
|
||||
from .lstr.lstr_trainer import *
|
||||
@@ -0,0 +1,2 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
@@ -0,0 +1,673 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
import os.path as osp
|
||||
import pickle as pkl
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
from rekognition_online_action_detection.datasets import build_dataset
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def batch_inference(cfg, model, device, logger, data_loader):
|
||||
|
||||
# Collect scores and targets
|
||||
pred_scores = {}
|
||||
pred_scores_verb = {}
|
||||
pred_scores_noun = {}
|
||||
gt_targets = {}
|
||||
|
||||
start_time = time.time()
|
||||
with torch.no_grad():
|
||||
pbar = tqdm(data_loader, desc='BatchInference')
|
||||
for batch_idx, data in enumerate(pbar, start=1):
|
||||
target = data[-4]
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
target, verb_target, noun_target = target
|
||||
|
||||
score = model(*[x.to(device) for x in data[:-4]])
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
score, score_verb, score_noun = score
|
||||
score = score.cpu().numpy()
|
||||
score_verb = score_verb.cpu().numpy()
|
||||
score_noun = score_noun.cpu().numpy()
|
||||
cfg.DATA.NUM_VERBS = 126 if cfg.DATA.DATA_NAME == 'EK55' else 98
|
||||
cfg.DATA.NUM_NOUNS = 353 if cfg.DATA.DATA_NAME == 'EK55' else 301
|
||||
else:
|
||||
score = (score.cpu().numpy()
|
||||
if cfg.DATA.DATA_NAME.startswith('EK')
|
||||
else score.softmax(dim=-1).cpu().numpy())
|
||||
for bs, (session, query_indices, num_frames) in enumerate(zip(*data[-3:])):
|
||||
# data[-3] is session, data[-2] is work_indices, data[-1] is num_frames
|
||||
if session not in pred_scores:
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
pred_scores[session] = np.zeros((num_frames, cfg.DATA.NUM_CLASSES, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES))
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
pred_scores_verb[session] = np.zeros((num_frames, cfg.DATA.NUM_VERBS, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES))
|
||||
pred_scores_noun[session] = np.zeros((num_frames, cfg.DATA.NUM_NOUNS, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES))
|
||||
else:
|
||||
pred_scores[session] = np.zeros((num_frames, cfg.DATA.NUM_CLASSES))
|
||||
if session not in gt_targets:
|
||||
gt_targets[session] = np.zeros((num_frames, cfg.DATA.NUM_CLASSES))
|
||||
|
||||
# DEFAULT VALUES in config
|
||||
# cfg.MODEL.LSTR.WORK_MEMORY_LENGTH: 40
|
||||
# cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE: 1
|
||||
# cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES: 10
|
||||
# query_indices.shape: torch.Size([cfg.MODEL.LSTR.WORK_MEMORY_LENGTH])
|
||||
# query_indices at first: [0, 1, 2, 3, ..., 39]
|
||||
# At first, go into this if statement
|
||||
if query_indices[0] in torch.arange(0, cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE):
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
# at first, full_indices indicate the range:
|
||||
# work_memory + t_a (anticiptaion timestep, from 0 to 9)
|
||||
full_indices = torch.cat((query_indices,
|
||||
torch.arange(query_indices[-1] + cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE,
|
||||
query_indices[-1] + (t_a + 1) + cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE)),
|
||||
dim=0)
|
||||
|
||||
# then assgin the currnet logits + anticipated logits to the pred_scores
|
||||
pred_scores[session][full_indices, :, t_a] = score[bs][:full_indices.shape[0]]
|
||||
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
pred_scores_verb[session][full_indices, :, t_a] = score_verb[bs][:full_indices.shape[0]]
|
||||
pred_scores_noun[session][full_indices, :, t_a] = score_noun[bs][:full_indices.shape[0]]
|
||||
|
||||
gt_targets[session][query_indices] = target[bs][:query_indices.shape[0]]
|
||||
else:
|
||||
# no anticipation
|
||||
pred_scores[session][query_indices] = score[bs]
|
||||
gt_targets[session][query_indices] = target[bs]
|
||||
else:
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
if query_indices[-1] + t_a + 1 < num_frames:
|
||||
pred_scores[session][query_indices[-1] + t_a + 1, :, t_a] = score[bs][t_a - cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES]
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
pred_scores_verb[session][query_indices[-1] + t_a + 1, :, t_a] = score_verb[bs][t_a - cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES]
|
||||
pred_scores_noun[session][query_indices[-1] + t_a + 1, :, t_a] = score_noun[bs][t_a - cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES]
|
||||
gt_targets[session][query_indices[-1]] = target[bs][-1]
|
||||
else:
|
||||
pred_scores[session][query_indices[-1]] = score[bs][-1]
|
||||
gt_targets[session][query_indices[-1]] = target[bs][-1]
|
||||
time_taken = time.time() - start_time
|
||||
# Save scores and targets
|
||||
# pkl.dump({
|
||||
# 'cfg': cfg,
|
||||
# 'perframe_pred_scores': pred_scores,
|
||||
# 'perframe_gt_targets': gt_targets,
|
||||
# }, open(osp.splitext(cfg.MODEL.CHECKPOINT)[0] + '.pkl', 'wb'))
|
||||
num_frames = np.concatenate(list(gt_targets.values()), axis=0).shape[0]
|
||||
logger.info(f'\nProcessed {num_frames} frames in {time_taken:.1f} seconds ({num_frames / time_taken :.1f} FPS)')
|
||||
logger.info('\n=============')
|
||||
# Compute results
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
maps_list = []
|
||||
per_class_list = {}
|
||||
|
||||
for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
result = compute_result['perframe'](
|
||||
cfg,
|
||||
np.concatenate(list(gt_targets.values()), axis=0),
|
||||
np.concatenate(list(pred_scores.values()), axis=0)[:, :, t_a],
|
||||
)
|
||||
|
||||
logger.info('Action anticipation ({:.2f}s) perframe m{}: {:.5f}'.format(
|
||||
(t_a + 1) / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE,
|
||||
cfg.DATA.METRICS, result['mean_AP']
|
||||
))
|
||||
maps_list.append(result['mean_AP'])
|
||||
# per_class_AP = result['per_class_AP']
|
||||
|
||||
anticipation_time=(t_a + 1) / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE
|
||||
|
||||
for class_name, ap in result['per_class_AP'].items():
|
||||
if class_name not in per_class_list:
|
||||
per_class_list[class_name] = []
|
||||
per_class_list[class_name].append(ap)
|
||||
logger.info(f'Action anticipation ({anticipation_time:.2f}s) perframe AP of {class_name}: {ap:.5f}')
|
||||
|
||||
|
||||
logger.info('Action anticipation (mean) perframe m{}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, np.mean(maps_list)
|
||||
))
|
||||
|
||||
for class_name, ap_list in per_class_list.items():
|
||||
logger.info(f'Action anticipation (mean) perframe AP of {class_name}: {np.mean(ap_list):.5f}')
|
||||
|
||||
# EDIT: save predictions!
|
||||
for session in pred_scores.keys():
|
||||
anticipation_score = pred_scores[session]
|
||||
anticipation_dir = os.path.join(cfg.OUTPUT_DIR, 'pred')
|
||||
if not os.path.exists(anticipation_dir):
|
||||
os.makedirs(anticipation_dir)
|
||||
|
||||
np.save(osp.join(anticipation_dir, session+'.npy'), anticipation_score)
|
||||
|
||||
else:
|
||||
result = compute_result['perframe'](
|
||||
cfg,
|
||||
np.concatenate(list(gt_targets.values()), axis=0),
|
||||
np.concatenate(list(pred_scores.values()), axis=0),
|
||||
)
|
||||
logger.info('Action detection perframe m{}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, result['mean_AP']
|
||||
))
|
||||
|
||||
# segment-level evaluation on EK55/EK100
|
||||
if cfg.DATA.DATA_NAME.startswith('EK'):
|
||||
try:
|
||||
from external.rulstm.RULSTM.utils import (
|
||||
get_marginal_indexes, marginalize, softmax,
|
||||
topk_accuracy_multiple_timesteps,
|
||||
topk_recall_multiple_timesteps, tta)
|
||||
except:
|
||||
raise ModuleNotFoundError
|
||||
if cfg.DATA.DATA_NAME == 'EK55':
|
||||
path_to_data = 'external/rulstm/RULSTM/data/ek55/'
|
||||
elif cfg.DATA.DATA_NAME == 'EK100':
|
||||
path_to_data = 'external/rulstm/RULSTM/data/ek100/'
|
||||
segment_list = pd.read_csv(osp.join(path_to_data, 'validation.csv'),
|
||||
names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
||||
skipinitialspace=True)
|
||||
predictions, labels, ids = [], [], []
|
||||
predictions_verb, predictions_noun = [], []
|
||||
discarded_labels, discarded_ids = [], []
|
||||
for segment in segment_list.iterrows():
|
||||
start_step = int(np.floor(segment[1]['start_f'] / 30 * cfg.DATA.FPS))
|
||||
vid = segment[1]['video'].strip()
|
||||
preds = (pred_scores[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
preds_v = (pred_scores_verb[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
preds_n = (pred_scores_noun[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
else:
|
||||
preds_v = None
|
||||
preds_n = None
|
||||
if start_step >= cfg.MODEL.LSTR.ANTICIPATION_LENGTH:
|
||||
# pt = []
|
||||
# for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_LENGTH):
|
||||
# pt.append(pred_scores[vid][start_step - t_a, 1:, t_a][None, None, :])
|
||||
# preds = np.concatenate(pt, axis=1)
|
||||
predictions.append(preds)
|
||||
if preds_v is not None:
|
||||
predictions_verb.append(preds_v)
|
||||
if preds_n is not None:
|
||||
predictions_noun.append(preds_n)
|
||||
labels.append(segment[1][['verb', 'noun', 'action']].values.astype(int))
|
||||
ids.append(segment[1]['id'])
|
||||
else:
|
||||
discarded_labels.append(segment[1][['verb', 'noun', 'action']].values.astype(int))
|
||||
discarded_ids.append(segment[1]['id'])
|
||||
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'), index_col='id')
|
||||
|
||||
action_scores = np.concatenate(predictions)
|
||||
if len(predictions_verb) > 0:
|
||||
verb_scores = np.concatenate(predictions_verb)
|
||||
else:
|
||||
verb_scores = None
|
||||
if len(predictions_noun) > 0:
|
||||
noun_scores = np.concatenate(predictions_noun)
|
||||
else:
|
||||
noun_scores = None
|
||||
labels = np.array(labels)
|
||||
ids = np.array(ids)
|
||||
|
||||
vi = get_marginal_indexes(actions, 'verb')
|
||||
ni = get_marginal_indexes(actions, 'noun')
|
||||
|
||||
pkl.dump({
|
||||
'cfg': cfg,
|
||||
'action_scores': action_scores,
|
||||
'verb_scores': verb_scores,
|
||||
'noun_scores': noun_scores,
|
||||
}, open(osp.splitext(cfg.MODEL.CHECKPOINT)[0] + '.ek55_style.pkl', 'wb'))
|
||||
|
||||
action_probs = softmax(action_scores.reshape(-1, action_scores.shape[-1]))
|
||||
# action_probs = action_probs / action_probs.sum(-1, keepdims=True)
|
||||
print(action_probs.shape)
|
||||
if verb_scores is None:
|
||||
verb_scores = marginalize(action_probs, vi).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
else:
|
||||
verb_scores = softmax(verb_scores.reshape(-1, verb_scores.shape[-1]))
|
||||
verb_scores = verb_scores.reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
if noun_scores is None:
|
||||
noun_scores = marginalize(action_probs, ni).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
else:
|
||||
noun_scores = softmax(noun_scores.reshape(-1, noun_scores.shape[-1]))
|
||||
noun_scores = noun_scores.reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
|
||||
include_discarded = (cfg.DATA.DATA_NAME == 'EK100')
|
||||
if include_discarded:
|
||||
dlab = np.array(discarded_labels)
|
||||
dislab = np.array(discarded_ids)
|
||||
ids = np.concatenate([ids, dislab])
|
||||
num_disc = len(dlab)
|
||||
labels = np.concatenate([labels, dlab])
|
||||
verb_scores = np.concatenate((verb_scores, np.zeros((num_disc, *verb_scores.shape[1:]))))
|
||||
noun_scores = np.concatenate((noun_scores, np.zeros((num_disc, *noun_scores.shape[1:]))))
|
||||
action_scores = np.concatenate((action_scores, np.zeros((num_disc, *action_scores.shape[1:]))))
|
||||
|
||||
verb_labels, noun_labels, action_labels = labels[:, 0], labels[:, 1], labels[:, 2]
|
||||
verb_accuracies = topk_accuracy_multiple_timesteps(verb_scores, verb_labels)
|
||||
noun_accuracies = topk_accuracy_multiple_timesteps(noun_scores, noun_labels)
|
||||
action_accuracies = topk_accuracy_multiple_timesteps(action_scores, action_labels)
|
||||
|
||||
if cfg.DATA.DATA_NAME == 'EK55':
|
||||
many_shot_verbs = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_verbs.csv'))['verb_class'].values
|
||||
many_shot_nouns = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_nouns.csv'))['noun_class'].values
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'))
|
||||
a_to_vn = {a[1]['id']: tuple(a[1][['verb', 'noun']].values)
|
||||
for a in actions.iterrows()}
|
||||
many_shot_actions = [a for a, (v, n) in a_to_vn.items()
|
||||
if v in many_shot_verbs or n in many_shot_nouns]
|
||||
|
||||
verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels,
|
||||
k=5, classes=many_shot_verbs)
|
||||
noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels,
|
||||
k=5, classes=many_shot_nouns)
|
||||
action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels,
|
||||
k=5, classes=many_shot_actions)
|
||||
|
||||
all_accuracies = np.concatenate([verb_accuracies, noun_accuracies, action_accuracies,
|
||||
verb_recalls, noun_recalls, action_recalls])
|
||||
all_accuracies = all_accuracies[[0, 1, 6, 2, 3, 7, 4, 5, 8]]
|
||||
indices = [
|
||||
('Verb', 'Top-1 Accuracy'), ('Verb', 'Top-5 Accuracy'),
|
||||
('Verb', 'Mean Top-5 Recall'),
|
||||
('Noun', 'Top-1 Accuracy'), ('Noun', 'Top-5 Accuracy'),
|
||||
('Noun', 'Mean Top-5 Recall'),
|
||||
('Action', 'Top-1 Accuracy'), ('Action', 'Top-5 Accuracy'),
|
||||
('Action', 'Mean Top-5 Recall'),
|
||||
]
|
||||
else:
|
||||
overall_verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels, k=5)
|
||||
overall_noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels, k=5)
|
||||
overall_action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels, k=5)
|
||||
|
||||
unseen = pd.read_csv(osp.join(path_to_data, 'validation_unseen_participants_ids.csv'), names=['id'], squeeze=True)
|
||||
tail_verbs = pd.read_csv(osp.join(path_to_data, 'validation_tail_verbs_ids.csv'), names=['id'], squeeze=True)
|
||||
tail_nouns = pd.read_csv(osp.join(path_to_data, 'validation_tail_nouns_ids.csv'), names=['id'], squeeze=True)
|
||||
tail_actions = pd.read_csv(osp.join(path_to_data, 'validation_tail_actions_ids.csv'), names=['id'], squeeze=True)
|
||||
|
||||
unseen_bool_idx = pd.Series(ids).isin(unseen).values
|
||||
tail_verbs_bool_idx = pd.Series(ids).isin(tail_verbs).values
|
||||
tail_nouns_bool_idx = pd.Series(ids).isin(tail_nouns).values
|
||||
tail_actions_bool_idx = pd.Series(ids).isin(tail_actions).values
|
||||
|
||||
tail_verb_recalls = topk_recall_multiple_timesteps(
|
||||
verb_scores[tail_verbs_bool_idx], verb_labels[tail_verbs_bool_idx], k=5)
|
||||
tail_noun_recalls = topk_recall_multiple_timesteps(
|
||||
noun_scores[tail_nouns_bool_idx], noun_labels[tail_nouns_bool_idx], k=5)
|
||||
tail_action_recalls = topk_recall_multiple_timesteps(
|
||||
action_scores[tail_actions_bool_idx], action_labels[tail_actions_bool_idx], k=5)
|
||||
|
||||
unseen_verb_recalls = topk_recall_multiple_timesteps(
|
||||
verb_scores[unseen_bool_idx], verb_labels[unseen_bool_idx], k=5)
|
||||
unseen_noun_recalls = topk_recall_multiple_timesteps(
|
||||
noun_scores[unseen_bool_idx], noun_labels[unseen_bool_idx], k=5)
|
||||
unseen_action_recalls = topk_recall_multiple_timesteps(
|
||||
action_scores[unseen_bool_idx], action_labels[unseen_bool_idx], k=5)
|
||||
|
||||
all_accuracies = np.concatenate(
|
||||
[overall_verb_recalls, overall_noun_recalls, overall_action_recalls,
|
||||
unseen_verb_recalls, unseen_noun_recalls, unseen_action_recalls,
|
||||
tail_verb_recalls, tail_noun_recalls, tail_action_recalls]
|
||||
) #9 x 8
|
||||
|
||||
indices = [
|
||||
('Overall Mean Top-5 Recall', 'Verb'), ('Overall Mean Top-5 Recall', 'Noun'),
|
||||
('Overall Mean Top-5 Recall', 'Action'),
|
||||
('Unseen Mean Top-5 Recall', 'Verb'), ('Unseen Mean Top-5 Recall', 'Noun'),
|
||||
('Unseen Mean Top-5 Recall', 'Action'),
|
||||
('Tail Mean Top-5 Recall', 'Verb'), ('Tail Mean Top-5 Recall', 'Noun'),
|
||||
('Tail Mean Top-5 Recall', 'Action'),
|
||||
]
|
||||
|
||||
cc = np.linspace(1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_LENGTH,
|
||||
1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE,
|
||||
cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES, dtype=str)
|
||||
scores = pd.DataFrame(all_accuracies * 100, columns=cc,
|
||||
index=pd.MultiIndex.from_tuples(indices))
|
||||
logger.info(scores)
|
||||
|
||||
tta_verb = tta(verb_scores, verb_labels)
|
||||
tta_noun = tta(noun_scores, noun_labels)
|
||||
tta_action = tta(action_scores, action_labels)
|
||||
logger.info(f'\nMean TtA(5): VERB {tta_verb:0.2f} '
|
||||
f'NOUN: {tta_noun:0.2f} ACTION: {tta_action:0.2f}')
|
||||
|
||||
|
||||
def do_perframe_det_batch_inference(cfg, model, device, logger):
|
||||
# Setup model to test mode
|
||||
model.eval()
|
||||
cfg.MODEL.CRITERIONS = [['MCE', {}]]
|
||||
|
||||
data_loader = torch.utils.data.DataLoader(
|
||||
dataset=build_dataset(cfg, phase='test', tag='BatchInference'),
|
||||
batch_size=cfg.DATA_LOADER.BATCH_SIZE,
|
||||
num_workers=cfg.DATA_LOADER.NUM_WORKERS,
|
||||
pin_memory=cfg.DATA_LOADER.PIN_MEMORY,
|
||||
)
|
||||
|
||||
logger.info(f"phase: test) Inference on test_dataloader")
|
||||
start = time.time()
|
||||
batch_inference(cfg, model, device, logger, data_loader)
|
||||
end = time.time() - start
|
||||
|
||||
logger.info(f"Elapsed time for function batch_inference: {end:.2f}s")
|
||||
|
||||
# data_loader = torch.utils.data.DataLoader(
|
||||
# dataset=build_dataset(cfg, phase='train', tag='BatchInference'),
|
||||
# batch_size=cfg.DATA_LOADER.BATCH_SIZE,
|
||||
# num_workers=cfg.DATA_LOADER.NUM_WORKERS,
|
||||
# pin_memory=cfg.DATA_LOADER.PIN_MEMORY,
|
||||
# )
|
||||
|
||||
# logger.info(f"***** phase: train")
|
||||
# batch_inference(cfg, model, device, logger, data_loader)
|
||||
|
||||
# # Collect scores and targets
|
||||
# pred_scores = {}
|
||||
# pred_scores_verb = {}
|
||||
# pred_scores_noun = {}
|
||||
# gt_targets = {}
|
||||
|
||||
# with torch.no_grad():
|
||||
# pbar = tqdm(data_loader, desc='BatchInference')
|
||||
# for batch_idx, data in enumerate(pbar, start=1):
|
||||
# target = data[-4]
|
||||
# if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
# target, verb_target, noun_target = target
|
||||
|
||||
# score = model(*[x.to(device) for x in data[:-4]])
|
||||
# if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
# score, score_verb, score_noun = score
|
||||
# score = score.cpu().numpy()
|
||||
# score_verb = score_verb.cpu().numpy()
|
||||
# score_noun = score_noun.cpu().numpy()
|
||||
# cfg.DATA.NUM_VERBS = 126 if cfg.DATA.DATA_NAME == 'EK55' else 98
|
||||
# cfg.DATA.NUM_NOUNS = 353 if cfg.DATA.DATA_NAME == 'EK55' else 301
|
||||
# else:
|
||||
# score = (score.cpu().numpy()
|
||||
# if cfg.DATA.DATA_NAME.startswith('EK')
|
||||
# else score.softmax(dim=-1).cpu().numpy())
|
||||
# for bs, (session, query_indices, num_frames) in enumerate(zip(*data[-3:])):
|
||||
# if session not in pred_scores:
|
||||
# if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
# pred_scores[session] = np.zeros((num_frames, cfg.DATA.NUM_CLASSES, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES))
|
||||
# if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
# pred_scores_verb[session] = np.zeros((num_frames, cfg.DATA.NUM_VERBS, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES))
|
||||
# pred_scores_noun[session] = np.zeros((num_frames, cfg.DATA.NUM_NOUNS, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES))
|
||||
# else:
|
||||
# pred_scores[session] = np.zeros((num_frames, cfg.DATA.NUM_CLASSES))
|
||||
# if session not in gt_targets:
|
||||
# gt_targets[session] = np.zeros((num_frames, cfg.DATA.NUM_CLASSES))
|
||||
|
||||
# if query_indices[0] in torch.arange(0, cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE):
|
||||
# if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
# for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
# full_indices = torch.cat((query_indices,
|
||||
# torch.arange(query_indices[-1] + cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE,
|
||||
# query_indices[-1] + t_a + 1 + cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE)),
|
||||
# dim=0)
|
||||
# pred_scores[session][full_indices, :, t_a] = score[bs][:full_indices.shape[0]]
|
||||
# if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
# pred_scores_verb[session][full_indices, :, t_a] = score_verb[bs][:full_indices.shape[0]]
|
||||
# pred_scores_noun[session][full_indices, :, t_a] = score_noun[bs][:full_indices.shape[0]]
|
||||
# gt_targets[session][query_indices] = target[bs][:query_indices.shape[0]]
|
||||
# else:
|
||||
# pred_scores[session][query_indices] = score[bs]
|
||||
# gt_targets[session][query_indices] = target[bs]
|
||||
# else:
|
||||
# if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
# for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
# if query_indices[-1] + t_a + 1 < num_frames:
|
||||
# pred_scores[session][query_indices[-1] + t_a + 1, :, t_a] = score[bs][t_a - cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES]
|
||||
# if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
# pred_scores_verb[session][query_indices[-1] + t_a + 1, :, t_a] = score_verb[bs][t_a - cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES]
|
||||
# pred_scores_noun[session][query_indices[-1] + t_a + 1, :, t_a] = score_noun[bs][t_a - cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES]
|
||||
# gt_targets[session][query_indices[-1]] = target[bs][-1]
|
||||
# else:
|
||||
# pred_scores[session][query_indices[-1]] = score[bs][-1]
|
||||
# gt_targets[session][query_indices[-1]] = target[bs][-1]
|
||||
|
||||
# # Save scores and targets
|
||||
# # pkl.dump({
|
||||
# # 'cfg': cfg,
|
||||
# # 'perframe_pred_scores': pred_scores,
|
||||
# # 'perframe_gt_targets': gt_targets,
|
||||
# # }, open(osp.splitext(cfg.MODEL.CHECKPOINT)[0] + '.pkl', 'wb'))
|
||||
|
||||
# # Compute results
|
||||
# if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
# maps_list = []
|
||||
# for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
# result = compute_result['perframe'](
|
||||
# cfg,
|
||||
# np.concatenate(list(gt_targets.values()), axis=0),
|
||||
# np.concatenate(list(pred_scores.values()), axis=0)[:, :, t_a],
|
||||
# )
|
||||
# logger.info('Action anticipation ({:.2f}s) perframe m{}: {:.5f}'.format(
|
||||
# (t_a + 1) / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE,
|
||||
# cfg.DATA.METRICS, result['mean_AP']
|
||||
# ))
|
||||
# maps_list.append(result['mean_AP'])
|
||||
# logger.info('Action anticipation (mean) perframe m{}: {:.5f}'.format(
|
||||
# cfg.DATA.METRICS, np.mean(maps_list)
|
||||
# ))
|
||||
# else:
|
||||
# result = compute_result['perframe'](
|
||||
# cfg,
|
||||
# np.concatenate(list(gt_targets.values()), axis=0),
|
||||
# np.concatenate(list(pred_scores.values()), axis=0),
|
||||
# )
|
||||
# logger.info('Action detection perframe m{}: {:.5f}'.format(
|
||||
# cfg.DATA.METRICS, result['mean_AP']
|
||||
# ))
|
||||
|
||||
# # segment-level evaluation on EK55/EK100
|
||||
# if cfg.DATA.DATA_NAME.startswith('EK'):
|
||||
# try:
|
||||
# from external.rulstm.RULSTM.utils import (get_marginal_indexes, marginalize, softmax,
|
||||
# topk_accuracy_multiple_timesteps,
|
||||
# topk_recall_multiple_timesteps,
|
||||
# tta)
|
||||
# except:
|
||||
# raise ModuleNotFoundError
|
||||
# if cfg.DATA.DATA_NAME == 'EK55':
|
||||
# path_to_data = 'external/rulstm/RULSTM/data/ek55/'
|
||||
# elif cfg.DATA.DATA_NAME == 'EK100':
|
||||
# path_to_data = 'external/rulstm/RULSTM/data/ek100/'
|
||||
# segment_list = pd.read_csv(osp.join(path_to_data, 'validation.csv'),
|
||||
# names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
||||
# skipinitialspace=True)
|
||||
# predictions, labels, ids = [], [], []
|
||||
# predictions_verb, predictions_noun = [], []
|
||||
# discarded_labels, discarded_ids = [], []
|
||||
# for segment in segment_list.iterrows():
|
||||
# start_step = int(np.floor(segment[1]['start_f'] / 30 * cfg.DATA.FPS))
|
||||
# vid = segment[1]['video'].strip()
|
||||
# preds = (pred_scores[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
# if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
# preds_v = (pred_scores_verb[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
# preds_n = (pred_scores_noun[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
# else:
|
||||
# preds_v = None
|
||||
# preds_n = None
|
||||
# if start_step >= cfg.MODEL.LSTR.ANTICIPATION_LENGTH:
|
||||
# # pt = []
|
||||
# # for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_LENGTH):
|
||||
# # pt.append(pred_scores[vid][start_step - t_a, 1:, t_a][None, None, :])
|
||||
# # preds = np.concatenate(pt, axis=1)
|
||||
# predictions.append(preds)
|
||||
# if preds_v is not None:
|
||||
# predictions_verb.append(preds_v)
|
||||
# if preds_n is not None:
|
||||
# predictions_noun.append(preds_n)
|
||||
# labels.append(segment[1][['verb', 'noun', 'action']].values.astype(int))
|
||||
# ids.append(segment[1]['id'])
|
||||
# else:
|
||||
# discarded_labels.append(segment[1][['verb', 'noun', 'action']].values.astype(int))
|
||||
# discarded_ids.append(segment[1]['id'])
|
||||
|
||||
# actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'), index_col='id')
|
||||
|
||||
# action_scores = np.concatenate(predictions)
|
||||
# if len(predictions_verb) > 0:
|
||||
# verb_scores = np.concatenate(predictions_verb)
|
||||
# else:
|
||||
# verb_scores = None
|
||||
# if len(predictions_noun) > 0:
|
||||
# noun_scores = np.concatenate(predictions_noun)
|
||||
# else:
|
||||
# noun_scores = None
|
||||
# labels = np.array(labels)
|
||||
# ids = np.array(ids)
|
||||
|
||||
# vi = get_marginal_indexes(actions, 'verb')
|
||||
# ni = get_marginal_indexes(actions, 'noun')
|
||||
|
||||
# pkl.dump({
|
||||
# 'cfg': cfg,
|
||||
# 'action_scores': action_scores,
|
||||
# 'verb_scores': verb_scores,
|
||||
# 'noun_scores': noun_scores,
|
||||
# }, open(osp.splitext(cfg.MODEL.CHECKPOINT)[0] + '.ek55_style.pkl', 'wb'))
|
||||
|
||||
# action_probs = softmax(action_scores.reshape(-1, action_scores.shape[-1]))
|
||||
# # action_probs = action_probs / action_probs.sum(-1, keepdims=True)
|
||||
# print(action_probs.shape)
|
||||
# if verb_scores is None:
|
||||
# verb_scores = marginalize(action_probs, vi).reshape(action_scores.shape[0],
|
||||
# action_scores.shape[1],
|
||||
# -1)
|
||||
# else:
|
||||
# verb_scores = softmax(verb_scores.reshape(-1, verb_scores.shape[-1]))
|
||||
# verb_scores = verb_scores.reshape(action_scores.shape[0],
|
||||
# action_scores.shape[1],
|
||||
# -1)
|
||||
# if noun_scores is None:
|
||||
# noun_scores = marginalize(action_probs, ni).reshape(action_scores.shape[0],
|
||||
# action_scores.shape[1],
|
||||
# -1)
|
||||
# else:
|
||||
# noun_scores = softmax(noun_scores.reshape(-1, noun_scores.shape[-1]))
|
||||
# noun_scores = noun_scores.reshape(action_scores.shape[0],
|
||||
# action_scores.shape[1],
|
||||
# -1)
|
||||
|
||||
# include_discarded = (cfg.DATA.DATA_NAME == 'EK100')
|
||||
# if include_discarded:
|
||||
# dlab = np.array(discarded_labels)
|
||||
# dislab = np.array(discarded_ids)
|
||||
# ids = np.concatenate([ids, dislab])
|
||||
# num_disc = len(dlab)
|
||||
# labels = np.concatenate([labels, dlab])
|
||||
# verb_scores = np.concatenate((verb_scores, np.zeros((num_disc, *verb_scores.shape[1:]))))
|
||||
# noun_scores = np.concatenate((noun_scores, np.zeros((num_disc, *noun_scores.shape[1:]))))
|
||||
# action_scores = np.concatenate((action_scores, np.zeros((num_disc, *action_scores.shape[1:]))))
|
||||
|
||||
# verb_labels, noun_labels, action_labels = labels[:, 0], labels[:, 1], labels[:, 2]
|
||||
# verb_accuracies = topk_accuracy_multiple_timesteps(verb_scores, verb_labels)
|
||||
# noun_accuracies = topk_accuracy_multiple_timesteps(noun_scores, noun_labels)
|
||||
# action_accuracies = topk_accuracy_multiple_timesteps(action_scores, action_labels)
|
||||
|
||||
# if cfg.DATA.DATA_NAME == 'EK55':
|
||||
# many_shot_verbs = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_verbs.csv'))['verb_class'].values
|
||||
# many_shot_nouns = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_nouns.csv'))['noun_class'].values
|
||||
# actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'))
|
||||
# a_to_vn = {a[1]['id']: tuple(a[1][['verb', 'noun']].values)
|
||||
# for a in actions.iterrows()}
|
||||
# many_shot_actions = [a for a, (v, n) in a_to_vn.items()
|
||||
# if v in many_shot_verbs or n in many_shot_nouns]
|
||||
|
||||
# verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels,
|
||||
# k=5, classes=many_shot_verbs)
|
||||
# noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels,
|
||||
# k=5, classes=many_shot_nouns)
|
||||
# action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels,
|
||||
# k=5, classes=many_shot_actions)
|
||||
|
||||
# all_accuracies = np.concatenate([verb_accuracies, noun_accuracies, action_accuracies,
|
||||
# verb_recalls, noun_recalls, action_recalls])
|
||||
# all_accuracies = all_accuracies[[0, 1, 6, 2, 3, 7, 4, 5, 8]]
|
||||
# indices = [
|
||||
# ('Verb', 'Top-1 Accuracy'), ('Verb', 'Top-5 Accuracy'),
|
||||
# ('Verb', 'Mean Top-5 Recall'),
|
||||
# ('Noun', 'Top-1 Accuracy'), ('Noun', 'Top-5 Accuracy'),
|
||||
# ('Noun', 'Mean Top-5 Recall'),
|
||||
# ('Action', 'Top-1 Accuracy'), ('Action', 'Top-5 Accuracy'),
|
||||
# ('Action', 'Mean Top-5 Recall'),
|
||||
# ]
|
||||
# else:
|
||||
# overall_verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels, k=5)
|
||||
# overall_noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels, k=5)
|
||||
# overall_action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels, k=5)
|
||||
|
||||
# unseen = pd.read_csv(osp.join(path_to_data, 'validation_unseen_participants_ids.csv'), names=['id'], squeeze=True)
|
||||
# tail_verbs = pd.read_csv(osp.join(path_to_data, 'validation_tail_verbs_ids.csv'), names=['id'], squeeze=True)
|
||||
# tail_nouns = pd.read_csv(osp.join(path_to_data, 'validation_tail_nouns_ids.csv'), names=['id'], squeeze=True)
|
||||
# tail_actions = pd.read_csv(osp.join(path_to_data, 'validation_tail_actions_ids.csv'), names=['id'], squeeze=True)
|
||||
|
||||
# unseen_bool_idx = pd.Series(ids).isin(unseen).values
|
||||
# tail_verbs_bool_idx = pd.Series(ids).isin(tail_verbs).values
|
||||
# tail_nouns_bool_idx = pd.Series(ids).isin(tail_nouns).values
|
||||
# tail_actions_bool_idx = pd.Series(ids).isin(tail_actions).values
|
||||
|
||||
# tail_verb_recalls = topk_recall_multiple_timesteps(
|
||||
# verb_scores[tail_verbs_bool_idx], verb_labels[tail_verbs_bool_idx], k=5)
|
||||
# tail_noun_recalls = topk_recall_multiple_timesteps(
|
||||
# noun_scores[tail_nouns_bool_idx], noun_labels[tail_nouns_bool_idx], k=5)
|
||||
# tail_action_recalls = topk_recall_multiple_timesteps(
|
||||
# action_scores[tail_actions_bool_idx], action_labels[tail_actions_bool_idx], k=5)
|
||||
|
||||
# unseen_verb_recalls = topk_recall_multiple_timesteps(
|
||||
# verb_scores[unseen_bool_idx], verb_labels[unseen_bool_idx], k=5)
|
||||
# unseen_noun_recalls = topk_recall_multiple_timesteps(
|
||||
# noun_scores[unseen_bool_idx], noun_labels[unseen_bool_idx], k=5)
|
||||
# unseen_action_recalls = topk_recall_multiple_timesteps(
|
||||
# action_scores[unseen_bool_idx], action_labels[unseen_bool_idx], k=5)
|
||||
|
||||
# all_accuracies = np.concatenate(
|
||||
# [overall_verb_recalls, overall_noun_recalls, overall_action_recalls,
|
||||
# unseen_verb_recalls, unseen_noun_recalls, unseen_action_recalls,
|
||||
# tail_verb_recalls, tail_noun_recalls, tail_action_recalls]
|
||||
# ) #9 x 8
|
||||
|
||||
# indices = [
|
||||
# ('Overall Mean Top-5 Recall', 'Verb'), ('Overall Mean Top-5 Recall', 'Noun'),
|
||||
# ('Overall Mean Top-5 Recall', 'Action'),
|
||||
# ('Unseen Mean Top-5 Recall', 'Verb'), ('Unseen Mean Top-5 Recall', 'Noun'),
|
||||
# ('Unseen Mean Top-5 Recall', 'Action'),
|
||||
# ('Tail Mean Top-5 Recall', 'Verb'), ('Tail Mean Top-5 Recall', 'Noun'),
|
||||
# ('Tail Mean Top-5 Recall', 'Action'),
|
||||
# ]
|
||||
|
||||
# cc = np.linspace(1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_LENGTH,
|
||||
# 1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE,
|
||||
# cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES, dtype=str)
|
||||
# scores = pd.DataFrame(all_accuracies * 100, columns=cc,
|
||||
# index=pd.MultiIndex.from_tuples(indices))
|
||||
# logger.info(scores)
|
||||
|
||||
# tta_verb = tta(verb_scores, verb_labels)
|
||||
# tta_noun = tta(noun_scores, noun_labels)
|
||||
# tta_action = tta(action_scores, action_labels)
|
||||
# logger.info(f'\nMean TtA(5): VERB {tta_verb:0.2f} '
|
||||
# f'NOUN: {tta_noun:0.2f} ACTION: {tta_action:0.2f}')
|
||||
# f'NOUN: {tta_noun:0.2f} ACTION: {tta_action:0.2f}')
|
||||
@@ -0,0 +1,2 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
@@ -0,0 +1,245 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
import os.path as osp
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import wandb
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def log_results(phase, cfg, compute_result, gt_targets, pred_scores, losses, data_loader, epoch=None, best_mAP=None, best_epoch=None):
|
||||
result = compute_result['perframe'](cfg, gt_targets, pred_scores)
|
||||
per_class_AP = result['mean_AP']
|
||||
|
||||
per_class_AP_log = ' '.join(f'{class_name}: {class_AP:.5f}' for class_name, class_AP in per_class_AP.items())
|
||||
|
||||
log = []
|
||||
if epoch is not None:
|
||||
log.append(f'\nEpoch {epoch:2}')
|
||||
|
||||
log.append(f'{phase} det_loss: {losses["det"] / len(data_loader.dataset):.5f} det_mAP: {result["mean_AP"]:.5f} {per_class_AP_log}')
|
||||
|
||||
if cfg.USE_WANDB:
|
||||
wandb.log({
|
||||
f"{phase.capitalize()} mAP": result['mean_AP'],
|
||||
f"{phase.capitalize()} per_class_AP": per_class_AP,
|
||||
f"{phase.capitalize()} Loss": losses['det'] / len(data_loader.dataset)
|
||||
})
|
||||
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
for loss_type in ['verb', 'noun']:
|
||||
if loss_type in losses:
|
||||
log.append(f'{phase} {loss_type}_loss: {losses[loss_type] / len(data_loader.dataset):.5f}')
|
||||
|
||||
return log, result['mean_AP']
|
||||
|
||||
def do_perframe_det_train(cfg,
|
||||
data_loaders,
|
||||
model,
|
||||
criterion,
|
||||
optimizer,
|
||||
scheduler,
|
||||
device,
|
||||
checkpointer,
|
||||
logger):
|
||||
# Setup model on multiple GPUs
|
||||
if torch.cuda.device_count() > 1:
|
||||
model = nn.DataParallel(model)
|
||||
|
||||
best_mAP, best_epoch = 0, 0
|
||||
best_per_class_AP_log = None
|
||||
|
||||
for epoch in range(cfg.SOLVER.START_EPOCH, cfg.SOLVER.START_EPOCH + cfg.SOLVER.NUM_EPOCHS):
|
||||
# Reset
|
||||
det_losses = {phase: 0.0 for phase in cfg.SOLVER.PHASES}
|
||||
pred_losses = {phase: 0.0 for phase in cfg.SOLVER.PHASES}
|
||||
verb_losses = {phase: 0.0 for phase in cfg.SOLVER.PHASES}
|
||||
noun_losses = {phase: 0.0 for phase in cfg.SOLVER.PHASES}
|
||||
det_pred_scores = []
|
||||
det_gt_targets = []
|
||||
aa_pred_scores = []
|
||||
aa_gt_targets = []
|
||||
|
||||
train_det_pred_scores = []
|
||||
train_det_gt_targets = []
|
||||
train_aa_pred_scores = []
|
||||
train_aa_gt_targets = []
|
||||
|
||||
start = time.time()
|
||||
|
||||
for phase in cfg.SOLVER.PHASES:
|
||||
training = phase == 'train'
|
||||
model.train(training)
|
||||
|
||||
with torch.set_grad_enabled(training):
|
||||
pbar = tqdm(data_loaders[phase],
|
||||
desc='{}ing epoch {}'.format(phase.capitalize(), epoch))
|
||||
for batch_idx, data in enumerate(pbar, start=1):
|
||||
batch_size = data[0].shape[0]
|
||||
|
||||
# batch_size 16
|
||||
# data[0]: visual_input, [16, 680, 2048]
|
||||
# [BATCH_SIZE, 640 + 40 (LONG_MEMORY_NUM_SAMPLES + WORK_MEMORY_NUM_SAMPLES), FEATURE_SIZE]
|
||||
# this if statement is for the case of EK100 dataset
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
det_target, verb_target, noun_target = data[-1]
|
||||
det_target = det_target.to(device)
|
||||
verb_target = verb_target.to(device)
|
||||
noun_target = noun_target.to(device)
|
||||
else:
|
||||
det_target = data[-1].to(device)
|
||||
|
||||
loss_names = list(zip(*cfg.MODEL.CRITERIONS))[0]
|
||||
if 'PRED_FUTURE' in list(zip(*cfg.MODEL.CRITERIONS))[0]:
|
||||
det_score, feat_out, feat_ori = model(*[x.to(device) for x in data[:-1]])
|
||||
else:
|
||||
det_score = model(*[x.to(device) for x in data[:-1]])
|
||||
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
det_score, verb_score, noun_score = det_score
|
||||
else:
|
||||
verb_score, noun_score = None, None
|
||||
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
aa_score = det_score[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
# [16, 10, 3]
|
||||
aa_target = det_target[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
|
||||
if cfg.MODEL.LSTR.LOSS_ANTICIPATE_ONLY:
|
||||
det_score = det_score[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
det_target = det_target[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
|
||||
# det_score: [16, 50, 3], aa_score: [16, 10, 3]
|
||||
# det_score can be different when aa_score
|
||||
|
||||
det_score = det_score.reshape(-1, cfg.DATA.NUM_CLASSES)
|
||||
det_target = det_target.reshape(-1, cfg.DATA.NUM_CLASSES)
|
||||
|
||||
assert len(det_score.shape) == 2
|
||||
det_loss = criterion[loss_names[0]](det_score, det_target)
|
||||
det_losses[phase] += det_loss.item() * batch_size
|
||||
|
||||
if 'PRED_FUTURE' in list(zip(*cfg.MODEL.CRITERIONS))[0]:
|
||||
pred_loss = criterion['PRED_FUTURE'](feat_out, feat_ori)
|
||||
pred_losses[phase] += pred_loss.item() * batch_size
|
||||
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
if cfg.MODEL.LSTR.LOSS_ANTICIPATE_ONLY:
|
||||
verb_score = verb_score[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
verb_target = verb_target[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
noun_score = noun_score[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
noun_target = noun_target[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
verb_score = verb_score.reshape(-1, verb_score.shape[-1])
|
||||
noun_score = noun_score.reshape(-1, noun_score.shape[-1])
|
||||
verb_target = verb_target.reshape(-1, verb_target.shape[-1])
|
||||
noun_target = noun_target.reshape(-1, noun_target.shape[-1])
|
||||
verb_loss = criterion['MCE'](verb_score, verb_target)
|
||||
noun_loss = criterion['MCE'](noun_score, noun_target)
|
||||
verb_losses[phase] += verb_loss.item() * batch_size
|
||||
noun_losses[phase] += noun_loss.item() * batch_size
|
||||
|
||||
# Output log for current batch
|
||||
pbar.set_postfix({
|
||||
'lr': '{:.7f}'.format(scheduler.get_last_lr()[0]),
|
||||
'det_loss': '{:.5f}'.format(det_loss.item()),
|
||||
})
|
||||
|
||||
if training:
|
||||
optimizer.zero_grad()
|
||||
loss = det_loss
|
||||
if 'PRED_FUTURE' in list(zip(*cfg.MODEL.CRITERIONS))[0]:
|
||||
loss += pred_loss
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
loss += noun_loss + verb_loss
|
||||
if loss.item() != 0:
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
scheduler.step()
|
||||
|
||||
if cfg.USE_WANDB:
|
||||
wandb.log({"Learning Rate": scheduler.get_last_lr()[0]})
|
||||
wandb.log({"Train Loss": det_loss.item()})
|
||||
|
||||
det_score = det_score.softmax(dim=1).cpu().tolist()
|
||||
det_target = det_target.cpu().tolist()
|
||||
train_det_pred_scores.extend(det_score)
|
||||
train_det_gt_targets.extend(det_target)
|
||||
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
aa_score = aa_score.softmax(dim=1).cpu().tolist()
|
||||
aa_target = aa_target.cpu().tolist()
|
||||
train_aa_pred_scores.extend(aa_score)
|
||||
train_aa_gt_targets.extend(aa_target)
|
||||
|
||||
else:
|
||||
# Prepare for evaluation
|
||||
det_score = det_score.softmax(dim=1).cpu().tolist()
|
||||
det_target = det_target.cpu().tolist()
|
||||
det_pred_scores.extend(det_score)
|
||||
det_gt_targets.extend(det_target)
|
||||
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
aa_score = aa_score.softmax(dim=1).cpu().tolist()
|
||||
aa_target = aa_target.cpu().tolist()
|
||||
aa_pred_scores.extend(aa_score)
|
||||
aa_gt_targets.extend(aa_target)
|
||||
|
||||
end = time.time()
|
||||
|
||||
# After the phase loop, compute and log results
|
||||
for phase in cfg.SOLVER.PHASES:
|
||||
if phase == 'train':
|
||||
gt_targets = train_det_gt_targets
|
||||
pred_scores = train_det_pred_scores
|
||||
else: # 'test'
|
||||
gt_targets = det_gt_targets
|
||||
pred_scores = det_pred_scores
|
||||
|
||||
result = compute_result['perframe'](cfg, gt_targets, pred_scores)
|
||||
per_class_AP = result['per_class_AP']
|
||||
|
||||
log = []
|
||||
log.append(f'\nEpoch {epoch:2}')
|
||||
log.append(f'{phase} det_loss: {det_losses[phase] / len(data_loaders[phase].dataset):.5f} det_mAP: {result["mean_AP"]:.5f}')
|
||||
|
||||
per_class_AP_log = ' '.join(f'{class_name}: {class_AP:.5f}' for class_name, class_AP in per_class_AP.items())
|
||||
log.append(f'Per-class AP: {per_class_AP_log}')
|
||||
|
||||
if cfg.USE_WANDB:
|
||||
wandb.log({
|
||||
f"{phase.capitalize()} mAP": result['mean_AP'],
|
||||
f"{phase.capitalize()} per_class_AP": per_class_AP,
|
||||
f"{phase.capitalize()} Loss": det_losses[phase] / len(data_loaders[phase].dataset)
|
||||
})
|
||||
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
log.append(f'{phase} verb_loss: {verb_losses[phase] / len(data_loaders[phase].dataset):.5f}')
|
||||
log.append(f'{phase} noun_loss: {noun_losses[phase] / len(data_loaders[phase].dataset):.5f}')
|
||||
|
||||
if phase == 'test':
|
||||
if result['mean_AP'] > best_mAP:
|
||||
best_mAP = result['mean_AP']
|
||||
best_epoch = epoch
|
||||
best_per_class_AP_log = per_class_AP_log
|
||||
checkpointer.save_best(epoch, model, optimizer)
|
||||
|
||||
log.append(f'\nbest epoch: {best_epoch:2} best det_mAP: {best_mAP:.5f} best per-class AP: {best_per_class_AP_log}\n')
|
||||
|
||||
logger.info(' | '.join(log))
|
||||
|
||||
log.append(f'running time: {end - start:.2f} sec')
|
||||
logger.info(' | '.join(log))
|
||||
|
||||
# Save checkpoint for model and optimizer
|
||||
if cfg.SOLVER.SAVE_EVERY_EPOCHS > 0 and epoch % cfg.SOLVER.SAVE_EVERY_EPOCHS == 0:
|
||||
checkpointer.save(epoch, model, optimizer)
|
||||
|
||||
# Shuffle dataset for next epoch
|
||||
data_loaders['train'].dataset.shuffle()
|
||||
@@ -0,0 +1,330 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
import os.path as osp
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import wandb
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def log_results(phase, cfg, compute_result, gt_targets, pred_scores, losses, data_loader, epoch=None, best_mAP=None, best_epoch=None):
|
||||
result = compute_result['perframe'](cfg, gt_targets, pred_scores)
|
||||
per_class_AP = result['mean_AP']
|
||||
|
||||
per_class_AP_log = ' '.join(f'{class_name}: {class_AP:.5f}' for class_name, class_AP in per_class_AP.items())
|
||||
|
||||
log = []
|
||||
if epoch is not None:
|
||||
log.append(f'\nEpoch {epoch:2}')
|
||||
|
||||
log.append(f'{phase} det_loss: {losses["det"] / len(data_loader.dataset):.5f} det_mAP: {result["mean_AP"]:.5f} {per_class_AP_log}')
|
||||
|
||||
if cfg.USE_WANDB:
|
||||
wandb.log({
|
||||
f"{phase.capitalize()} mAP": result['mean_AP'],
|
||||
f"{phase.capitalize()} per_class_AP": per_class_AP,
|
||||
f"{phase.capitalize()} Loss": losses['det'] / len(data_loader.dataset)
|
||||
})
|
||||
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
for loss_type in ['verb', 'noun']:
|
||||
if loss_type in losses:
|
||||
log.append(f'{phase} {loss_type}_loss: {losses[loss_type] / len(data_loader.dataset):.5f}')
|
||||
|
||||
return log, result['mean_AP']
|
||||
|
||||
def do_perframe_det_train_rnn(cfg,
|
||||
data_loaders,
|
||||
model,
|
||||
criterion,
|
||||
optimizer,
|
||||
scheduler,
|
||||
device,
|
||||
checkpointer,
|
||||
logger):
|
||||
# Setup model on multiple GPUs
|
||||
if torch.cuda.device_count() > 1:
|
||||
model = nn.DataParallel(model)
|
||||
|
||||
best_mAP, best_epoch = 0, 0
|
||||
best_per_class_AP_log = None
|
||||
|
||||
for epoch in range(cfg.SOLVER.START_EPOCH, cfg.SOLVER.START_EPOCH + cfg.SOLVER.NUM_EPOCHS):
|
||||
# Reset
|
||||
det_losses = {phase: 0.0 for phase in cfg.SOLVER.PHASES}
|
||||
pred_losses = {phase: 0.0 for phase in cfg.SOLVER.PHASES}
|
||||
|
||||
det_pred_scores = []
|
||||
det_gt_targets = []
|
||||
aa_pred_scores = []
|
||||
aa_gt_targets = []
|
||||
|
||||
train_det_pred_scores = []
|
||||
train_det_gt_targets = []
|
||||
train_aa_pred_scores = []
|
||||
train_aa_gt_targets = []
|
||||
|
||||
start = time.time()
|
||||
|
||||
for phase in cfg.SOLVER.PHASES:
|
||||
training = phase == 'train'
|
||||
model.train(training)
|
||||
|
||||
with torch.set_grad_enabled(training):
|
||||
pbar = tqdm(data_loaders[phase],
|
||||
desc='{}ing epoch {}'.format(phase.capitalize(), epoch))
|
||||
|
||||
# __getitem in LSTR:
|
||||
# return (fusion_visual_inputs, fusion_motion_inputs,
|
||||
# fusion_object_inputs, memory_key_padding_mask, target)
|
||||
# __getitem__ in RNN:
|
||||
# vid, rgb_input, flow_input, text_input, target, ant_target
|
||||
for batch_idx, data in enumerate(pbar, start=1):
|
||||
batch_size = data[1].shape[0]
|
||||
det_target = data[-2].to(device)
|
||||
aa_target = data[-1].to(device)
|
||||
|
||||
loss_names = list(zip(*cfg.MODEL.CRITERIONS))[0]
|
||||
assert 'PRED_FUTURE' not in list(zip(*cfg.MODEL.CRITERIONS))[0]
|
||||
# if 'PRED_FUTURE' in list(zip(*cfg.MODEL.CRITERIONS))[0]:
|
||||
# det_score, feat_out, feat_ori = model(*[x.to(device) for x in data[:-1]])
|
||||
# else:
|
||||
# det_score = model(*[x.to(device) for x in data[1:4]])
|
||||
|
||||
out_dict = model(*[x.to(device) for x in data[1:4]])
|
||||
# out_dict['logits']: [B, window_size, num_classes]
|
||||
# out_dict['anticipation_logits']: [B, window_size, ant_len, num_classes]
|
||||
|
||||
# [workmem_len + ant_len, batch_size, self.d_model]: [50, 16, 1024]
|
||||
|
||||
det_score = out_dict['logits']
|
||||
det_score = det_score
|
||||
if cfg.MODEL.MINIROAD.ANTICIPATION_LENGTH > 0:
|
||||
aa_score = out_dict['anticipation_logits'] # [B, window_size, anticipation_length, num_class]
|
||||
aa_score = aa_score[:, -1, :, :].reshape(-1, cfg.DATA.NUM_CLASSES)
|
||||
|
||||
aa_target = aa_target.reshape(-1, cfg.DATA.NUM_CLASSES)
|
||||
det_loss = criterion[loss_names[0]](aa_score, aa_target)
|
||||
anticipation_logits = out_dict['anticipation_logits']
|
||||
|
||||
else:
|
||||
det_score = det_score.reshape(-1, cfg.DATA.NUM_CLASSES)
|
||||
det_target = det_target.reshape(-1, cfg.DATA.NUM_CLASSES)
|
||||
det_loss = criterion[loss_names[0]](det_score, det_target)
|
||||
|
||||
det_loss = criterion[loss_names[0]](det_score, det_target)
|
||||
|
||||
|
||||
# if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
# aa_score = det_score[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
# # [16, 10, 3]
|
||||
# aa_target = det_target[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
|
||||
# if cfg.MODEL.LSTR.LOSS_ANTICIPATE_ONLY:
|
||||
# det_score = det_score[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
# det_target = det_target[:, -cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:, :]
|
||||
|
||||
# det_score: [16, 50, 3], aa_score: [16, 10, 3]
|
||||
# det_score can be different when aa_score
|
||||
|
||||
det_losses[phase] += det_loss.item() * batch_size
|
||||
|
||||
# Output log for current batch
|
||||
pbar.set_postfix({
|
||||
'lr': '{:.7f}'.format(scheduler.get_last_lr()[0]),
|
||||
'det_loss': '{:.5f}'.format(det_loss.item()),
|
||||
})
|
||||
|
||||
if training:
|
||||
optimizer.zero_grad()
|
||||
loss = det_loss
|
||||
if 'PRED_FUTURE' in list(zip(*cfg.MODEL.CRITERIONS))[0]:
|
||||
loss += pred_loss
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
loss += noun_loss + verb_loss
|
||||
if loss.item() != 0:
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
scheduler.step()
|
||||
|
||||
if cfg.USE_WANDB:
|
||||
wandb.log({"Learning Rate": scheduler.get_last_lr()[0]})
|
||||
wandb.log({"Train Loss": det_loss.item()})
|
||||
|
||||
det_score = det_score.softmax(dim=1).cpu().tolist()
|
||||
det_target = det_target.cpu().tolist()
|
||||
train_det_pred_scores.extend(det_score)
|
||||
train_det_gt_targets.extend(det_target)
|
||||
# len(train_det_pred_scores) + 16
|
||||
|
||||
if cfg.MODEL.MINIROAD.ANTICIPATION_LENGTH > 0:
|
||||
aa_score = aa_score.softmax(dim=1).cpu().tolist()
|
||||
aa_target = aa_target.cpu().tolist()
|
||||
train_aa_pred_scores.extend(aa_score)
|
||||
train_aa_gt_targets.extend(aa_target)
|
||||
|
||||
else:
|
||||
# Prepare for evaluation
|
||||
det_score = det_score.softmax(dim=1).cpu().tolist()
|
||||
det_target = det_target.cpu().tolist()
|
||||
det_pred_scores.extend(det_score)
|
||||
det_gt_targets.extend(det_target)
|
||||
|
||||
if cfg.MODEL.MINIROAD.ANTICIPATION_LENGTH > 0:
|
||||
aa_score = aa_score.softmax(dim=1).cpu().tolist()
|
||||
aa_target = aa_target.cpu().tolist()
|
||||
aa_pred_scores.extend(aa_score)
|
||||
aa_gt_targets.extend(aa_target)
|
||||
|
||||
# print(f'len(det_score): {len(det_score)}, len(det_pred_scores): {len(det_pred_scores)}')
|
||||
# print(f'len(aa_score): {len(aa_score)}, len(aa_pred_scores): {len(aa_pred_scores)}')
|
||||
|
||||
end = time.time()
|
||||
|
||||
# After the phase loop, compute and log results
|
||||
for phase in cfg.SOLVER.PHASES:
|
||||
if phase == 'train':
|
||||
gt_targets = train_det_gt_targets
|
||||
pred_scores = train_det_pred_scores
|
||||
else: # 'test'
|
||||
gt_targets = det_gt_targets
|
||||
pred_scores = det_pred_scores
|
||||
|
||||
result = compute_result['perframe'](cfg, gt_targets, pred_scores)
|
||||
per_class_AP = result['per_class_AP']
|
||||
|
||||
log = []
|
||||
log.append(f'\nEpoch {epoch:2}')
|
||||
log.append(f'{phase} det_loss: {det_losses[phase] / len(data_loaders[phase].dataset):.5f} det_mAP: {result["mean_AP"]:.5f}')
|
||||
|
||||
per_class_AP_log = ' '.join(f'{class_name}: {class_AP:.5f}' for class_name, class_AP in per_class_AP.items())
|
||||
log.append(f'Per-class AP: {per_class_AP_log}')
|
||||
|
||||
if cfg.USE_WANDB:
|
||||
wandb.log({
|
||||
f"{phase.capitalize()} mAP": result['mean_AP'],
|
||||
f"{phase.capitalize()} per_class_AP": per_class_AP,
|
||||
f"{phase.capitalize()} Loss": det_losses[phase] / len(data_loaders[phase].dataset)
|
||||
})
|
||||
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
log.append(f'{phase} verb_loss: {verb_losses[phase] / len(data_loaders[phase].dataset):.5f}')
|
||||
log.append(f'{phase} noun_loss: {noun_losses[phase] / len(data_loaders[phase].dataset):.5f}')
|
||||
|
||||
if phase == 'test':
|
||||
if result['mean_AP'] > best_mAP:
|
||||
best_mAP = result['mean_AP']
|
||||
best_epoch = epoch
|
||||
best_per_class_AP_log = per_class_AP_log
|
||||
checkpointer.save_best(epoch, model, optimizer)
|
||||
|
||||
log.append(f'\nbest epoch: {best_epoch:2} best det_mAP: {best_mAP:.5f} best per-class AP: {best_per_class_AP_log}\n')
|
||||
|
||||
logger.info(' | '.join(log))
|
||||
|
||||
log.append(f'running time: {end - start:.2f} sec')
|
||||
logger.info(' | '.join(log))
|
||||
|
||||
# Save checkpoint for model and optimizer
|
||||
if cfg.SOLVER.SAVE_EVERY_EPOCHS > 0 and epoch % cfg.SOLVER.SAVE_EVERY_EPOCHS == 0:
|
||||
checkpointer.save(epoch, model, optimizer)
|
||||
|
||||
# Shuffle dataset for next epoch
|
||||
data_loaders['train'].dataset.shuffle()
|
||||
|
||||
# @TRAINER.register("OAD")
|
||||
def train_one_epoch(trainloader, model, criterion, optimizer, scaler, epoch, use_wandb = False, writer=None, scheduler=None):
|
||||
epoch_loss = 0
|
||||
for it, (vid_name, rgb_input, flow_input, target) in enumerate(tqdm(trainloader, desc=f'Epoch:{epoch} Training', postfix=f'lr: {optimizer.param_groups[0]["lr"]:.7f}')):
|
||||
rgb_input, flow_input, target = rgb_input.cuda(), flow_input.cuda(), target.cuda()
|
||||
model.train()
|
||||
if scaler != None:
|
||||
with torch.cuda.amp.autocast():
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
scaler.scale(loss).backward()
|
||||
scaler.step(optimizer)
|
||||
scaler.update()
|
||||
else:
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
epoch_loss += loss.item()
|
||||
if use_wandb == True:
|
||||
wandb.log({"Train Loss": loss.item()})
|
||||
|
||||
if writer != None:
|
||||
writer.add_scalar("Train Loss", loss.item(), it+epoch*len(trainloader))
|
||||
return epoch_loss
|
||||
|
||||
# @TRAINER.register("OADTR")
|
||||
def train_one_epoch_oadtr(trainloader, model, criterion, optimizer, scaler, epoch, use_wandb = False, writer=None, scheduler=None):
|
||||
epoch_loss = 0
|
||||
for it, (vid_name, rgb_input, flow_input, enc_target, distance_target, class_h_target, dec_target) in enumerate(tqdm(trainloader, desc=f'Epoch:{epoch} Training', postfix=f'lr: {optimizer.param_groups[0]["lr"]:.7f}')):
|
||||
rgb_input, flow_input, enc_target, distance_target, class_h_target = rgb_input.cuda(), flow_input.cuda(), enc_target.cuda(), distance_target.cuda(), class_h_target.cuda()
|
||||
model.train()
|
||||
if scaler != None:
|
||||
with torch.cuda.amp.autocast():
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, enc_target, dec_target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
scaler.scale(loss).backward()
|
||||
scaler.step(optimizer)
|
||||
scaler.update()
|
||||
else:
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, enc_target, dec_target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
epoch_loss += loss.item()
|
||||
if use_wandb == True:
|
||||
wandb.log({"Train Loss": loss.item()})
|
||||
|
||||
if writer != None:
|
||||
writer.add_scalar("Train Loss", loss.item(), it+epoch*len(trainloader))
|
||||
|
||||
return epoch_loss
|
||||
|
||||
|
||||
# @TRAINER.register("ANTICIPATION")
|
||||
def ant_train_one_epoch(trainloader, model, criterion, optimizer, scaler, epoch, use_wandb = False, writer=None, scheduler=None):
|
||||
epoch_loss = 0
|
||||
for it, (vid_name, rgb_input, flow_input, text_input, target, ant_target) in enumerate(tqdm(trainloader, desc=f'Epoch:{epoch} Training', postfix=f'lr: {optimizer.param_groups[0]["lr"]:.7f}')):
|
||||
rgb_input, flow_input, text_input, target, ant_target = rgb_input.cuda(), flow_input.cuda(), text_input.cuda(), target.cuda(), ant_target.cuda()
|
||||
model.train()
|
||||
if scaler != None:
|
||||
with torch.cuda.amp.autocast():
|
||||
out_dict = model(rgb_input, flow_input, text_input)
|
||||
loss = criterion(out_dict, target, ant_target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
scaler.scale(loss).backward()
|
||||
scaler.step(optimizer)
|
||||
scaler.update()
|
||||
else:
|
||||
out_dict = model(rgb_input, flow_input, text_input)
|
||||
loss = criterion(out_dict, target, ant_target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
epoch_loss += loss.item()
|
||||
|
||||
if use_wandb == True:
|
||||
wandb.log({"Train Loss": loss.item()})
|
||||
|
||||
if writer != None:
|
||||
writer.add_scalar("Train Loss", loss.item(), it+epoch*len(trainloader))
|
||||
|
||||
return epoch_loss
|
||||
39
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/engines.py
vendored
Normal file
39
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/engines.py
vendored
Normal file
@@ -0,0 +1,39 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from rekognition_online_action_detection.utils.registry import Registry
|
||||
|
||||
TRAINERS = Registry()
|
||||
INFERENCES = Registry()
|
||||
|
||||
def do_train(cfg,
|
||||
data_loaders,
|
||||
model,
|
||||
criterion,
|
||||
optimizer,
|
||||
scheduler,
|
||||
device,
|
||||
checkpointer,
|
||||
logger):
|
||||
|
||||
return TRAINERS[cfg.MODEL.MODEL_NAME](
|
||||
cfg,
|
||||
data_loaders,
|
||||
model,
|
||||
criterion,
|
||||
optimizer,
|
||||
scheduler,
|
||||
device,
|
||||
checkpointer,
|
||||
logger)
|
||||
|
||||
|
||||
def do_inference(cfg,
|
||||
model,
|
||||
device,
|
||||
logger):
|
||||
return INFERENCES[cfg.MODEL.MODEL_NAME](
|
||||
cfg,
|
||||
model,
|
||||
device,
|
||||
logger)
|
||||
2
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/lstr/__init__.py
vendored
Normal file
2
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/lstr/__init__.py
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
417
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/lstr/lstr_inference.py
vendored
Normal file
417
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/lstr/lstr_inference.py
vendored
Normal file
@@ -0,0 +1,417 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os.path as osp
|
||||
from bisect import bisect_right
|
||||
import time
|
||||
import pickle as pkl
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
|
||||
from ..base_inferences.perframe_det_batch_inference import do_perframe_det_batch_inference
|
||||
|
||||
|
||||
from ..engines import INFERENCES as registry
|
||||
|
||||
|
||||
@registry.register('LSTR')
|
||||
def do_lstr_batch_inference(cfg,
|
||||
model,
|
||||
device,
|
||||
logger):
|
||||
if cfg.MODEL.LSTR.INFERENCE_MODE == 'stream':
|
||||
do_lstr_stream_inference(cfg,
|
||||
model,
|
||||
device,
|
||||
logger)
|
||||
else:
|
||||
do_perframe_det_batch_inference(cfg,
|
||||
model,
|
||||
device,
|
||||
logger)
|
||||
|
||||
|
||||
def do_lstr_stream_inference(cfg, model, device, logger):
|
||||
# Setup model to test mode
|
||||
model.eval()
|
||||
|
||||
# Collect scores and targets
|
||||
pred_scores = []
|
||||
gt_targets = []
|
||||
|
||||
def to_device(x, dtype=np.float32):
|
||||
return torch.as_tensor(x.astype(dtype)).unsqueeze(0).to(device)
|
||||
|
||||
long_memory_length = cfg.MODEL.LSTR.LONG_MEMORY_LENGTH
|
||||
long_memory_sample_rate = cfg.MODEL.LSTR.LONG_MEMORY_SAMPLE_RATE
|
||||
long_memory_num_samples = cfg.MODEL.LSTR.LONG_MEMORY_NUM_SAMPLES
|
||||
work_memory_length = cfg.MODEL.LSTR.WORK_MEMORY_LENGTH
|
||||
work_memory_sample_rate = cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE
|
||||
work_memory_num_samples = cfg.MODEL.LSTR.WORK_MEMORY_NUM_SAMPLES
|
||||
|
||||
# if len(cfg.DATA.TEST_SESSION_SET) != 1:
|
||||
# raise RuntimeError('Only support testing one video each time for stream inference, will fix later')
|
||||
|
||||
pred_scores_all = {}
|
||||
gt_targets_all = {}
|
||||
|
||||
with torch.no_grad():
|
||||
for session_idx, session in enumerate(cfg.DATA.TEST_SESSION_SET):
|
||||
model.clear_cache()
|
||||
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
pred_scores = [[] for _ in range(cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES)]
|
||||
else:
|
||||
pred_scores = []
|
||||
gt_targets = []
|
||||
|
||||
visual_inputs = np.load(osp.join(cfg.DATA.DATA_ROOT, cfg.INPUT.VISUAL_FEATURE, session + '.npy'), mmap_mode='r')
|
||||
motion_inputs = np.load(osp.join(cfg.DATA.DATA_ROOT, cfg.INPUT.MOTION_FEATURE, session + '.npy'), mmap_mode='r')
|
||||
object_inputs = np.load(osp.join(cfg.DATA.DATA_ROOT, cfg.INPUT.OBJECT_FEATURE, session + '.npy'), mmap_mode='r')
|
||||
target = np.load(osp.join(cfg.DATA.DATA_ROOT, cfg.INPUT.TARGET_PERFRAME, session + '.npy'))
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
for work_start, work_end in zip(range(0, target.shape[0] + 1),
|
||||
range(work_memory_length, target.shape[0] + 1)):
|
||||
# Get target
|
||||
# target = target[::work_memory_sample_rate]
|
||||
|
||||
# Get work memory
|
||||
work_indices = np.arange(work_start, work_end).clip(0)
|
||||
work_indices = work_indices[::work_memory_sample_rate]
|
||||
work_visual_inputs = to_device(visual_inputs[work_indices])
|
||||
work_motion_inputs = to_device(motion_inputs[work_indices])
|
||||
work_object_inputs = to_device(object_inputs[work_indices])
|
||||
|
||||
# Get long memory
|
||||
PRECISE = True
|
||||
if PRECISE:
|
||||
long_end = work_start - long_memory_sample_rate
|
||||
if long_end < 0:
|
||||
long_indices = [0 for _ in range(long_memory_num_samples)]
|
||||
long_indices_set = [long_indices for _ in range(long_memory_sample_rate)]
|
||||
long_visual_inputs = to_device(visual_inputs[long_indices])
|
||||
long_motion_inputs = to_device(motion_inputs[long_indices])
|
||||
long_object_inputs = to_device(object_inputs[long_indices])
|
||||
else:
|
||||
long_indices = long_indices_set[long_end % long_memory_sample_rate][1:] + [long_end]
|
||||
long_indices_set[long_end % long_memory_sample_rate] = long_indices
|
||||
long_visual_inputs = to_device(visual_inputs[[long_end]])
|
||||
long_motion_inputs = to_device(motion_inputs[[long_end]])
|
||||
long_object_inputs = to_device(object_inputs[[long_end]])
|
||||
else:
|
||||
long_end = work_start - 1
|
||||
if long_end == -1:
|
||||
long_indices = [0 for _ in range(long_memory_num_samples)]
|
||||
long_visual_inputs = to_device(visual_inputs[long_indices])
|
||||
long_motion_inputs = to_device(motion_inputs[long_indices])
|
||||
long_object_inputs = to_device(object_inputs[long_indices])
|
||||
elif long_end % long_memory_sample_rate == 0:
|
||||
long_indices = long_indices[1:] + [long_end]
|
||||
long_visual_inputs = to_device(visual_inputs[[long_end]])
|
||||
long_motion_inputs = to_device(motion_inputs[[long_end]])
|
||||
long_object_inputs = to_device(object_inputs[[long_end]])
|
||||
else:
|
||||
long_visual_inputs = None
|
||||
long_motion_inputs = None
|
||||
long_object_inputs = None
|
||||
|
||||
# Get memory key padding mask
|
||||
memory_key_padding_mask = np.zeros(len(long_indices))
|
||||
last_zero = bisect_right(long_indices, 0) - 1
|
||||
if last_zero > 0:
|
||||
memory_key_padding_mask[:last_zero] = float('-inf')
|
||||
memory_key_padding_mask = torch.as_tensor(memory_key_padding_mask.astype(np.float32)).unsqueeze(0).to(device)
|
||||
|
||||
score = model.stream_inference(
|
||||
long_visual_inputs,
|
||||
long_motion_inputs,
|
||||
long_object_inputs,
|
||||
work_visual_inputs,
|
||||
work_motion_inputs,
|
||||
work_object_inputs,
|
||||
memory_key_padding_mask,
|
||||
cache_num=long_memory_sample_rate if PRECISE else 1,
|
||||
cache_id=long_end % long_memory_sample_rate if PRECISE else 0)[0]
|
||||
|
||||
if cfg.DATA.DATA_NAME.startswith('EK'):
|
||||
score = score.cpu().numpy()
|
||||
else:
|
||||
score = score.softmax(dim=-1).cpu().numpy()
|
||||
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
if work_start == 0:
|
||||
upsample_score = score[:-cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES].repeat(work_memory_sample_rate, axis=0)
|
||||
upsample_score = upsample_score[work_start:work_end-1]
|
||||
anticipate_score = score[-cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES:]
|
||||
# coarse but sufficient
|
||||
for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
combined_score = np.concatenate((upsample_score, anticipate_score[:t_a + 1]),
|
||||
axis=0)
|
||||
pred_scores[t_a].extend(list(combined_score))
|
||||
gt_targets.extend(list(target[:work_end-1]))
|
||||
else:
|
||||
for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
pred_scores[t_a].append(list(score[t_a - cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES]))
|
||||
gt_targets.append(list(target[work_end - 1]))
|
||||
else:
|
||||
if work_start == 0:
|
||||
gt_targets.extend(list(target[:work_end]))
|
||||
pred_scores.extend(list(score))
|
||||
else:
|
||||
gt_targets.append(list(target[work_end - 1]))
|
||||
pred_scores.append(list(score[-1]))
|
||||
|
||||
end_time = time.time()
|
||||
logger.info('Running time: {:.3f} seconds'.format(end_time - start_time))
|
||||
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
result = compute_result['perframe'](
|
||||
cfg,
|
||||
gt_targets,
|
||||
pred_scores[t_a][:-1 - t_a],
|
||||
)
|
||||
sec = (t_a + 1) / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE
|
||||
logger.info('mAP of video ({:.2f}s) {}: {:.5f}'.format(sec, session, result['mean_AP']))
|
||||
else:
|
||||
result = compute_result['perframe'](
|
||||
cfg,
|
||||
gt_targets,
|
||||
pred_scores,
|
||||
)
|
||||
logger.info('mAP of video {}: {:.5f}'.format(session, result['mean_AP']))
|
||||
|
||||
|
||||
gt_targets_all[session] = np.array(gt_targets)
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
pred_scores[t_a] = np.array(pred_scores[t_a][:- 1 - t_a])
|
||||
pred_scores_all[session] = np.stack(pred_scores, axis=0).transpose(1, 2, 0)
|
||||
pred_scores_all[session] = pred_scores_all[session]
|
||||
else:
|
||||
pred_scores_all[session] = np.array(pred_scores)
|
||||
|
||||
|
||||
# pkl.dump({
|
||||
# 'cfg': cfg,
|
||||
# 'perframe_pred_scores': pred_scores_all,
|
||||
# 'perframe_gt_targets': gt_targets_all,
|
||||
# }, open(osp.splitext(cfg.MODEL.CHECKPOINT)[0] + '.stream.pkl', 'wb'))
|
||||
|
||||
if cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES > 0:
|
||||
maps_list = []
|
||||
for t_a in range(0, cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES):
|
||||
result = compute_result['perframe'](
|
||||
cfg,
|
||||
np.concatenate(list(gt_targets_all.values()), axis=0),
|
||||
np.concatenate(list(pred_scores_all.values()), axis=0)[:, :, t_a],
|
||||
)
|
||||
logger.info('Action anticipation ({:.2f}s) perframe m{}: {:.5f}'.format(
|
||||
(t_a + 1) / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE,
|
||||
cfg.DATA.METRICS, result['mean_AP']
|
||||
))
|
||||
maps_list.append(result['mean_AP'])
|
||||
logger.info('Action anticipation (mean) perframe m{}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, np.mean(maps_list)
|
||||
))
|
||||
else:
|
||||
result = compute_result['perframe'](
|
||||
cfg,
|
||||
np.concatenate(list(gt_targets_all.values()), axis=0),
|
||||
np.concatenate(list(pred_scores_all.values()), axis=0),
|
||||
)
|
||||
logger.info('Action detection perframe m{}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, result['mean_AP']
|
||||
))
|
||||
|
||||
gt_targets = gt_targets_all
|
||||
pred_scores = pred_scores_all
|
||||
pred_scores_verb = {}
|
||||
pred_scores_noun = {}
|
||||
|
||||
# segment-level evaluation on EK55/EK100
|
||||
if cfg.DATA.DATA_NAME.startswith('EK'):
|
||||
try:
|
||||
import pandas as pd
|
||||
from external.rulstm.RULSTM.utils import (get_marginal_indexes, marginalize, softmax,
|
||||
topk_accuracy_multiple_timesteps,
|
||||
topk_recall_multiple_timesteps,
|
||||
tta)
|
||||
except:
|
||||
raise ModuleNotFoundError
|
||||
if cfg.DATA.DATA_NAME == 'EK55':
|
||||
path_to_data = 'external/rulstm/RULSTM/data/ek55/'
|
||||
elif cfg.DATA.DATA_NAME == 'EK100':
|
||||
path_to_data = 'external/rulstm/RULSTM/data/ek100/'
|
||||
segment_list = pd.read_csv(osp.join(path_to_data, 'validation.csv'),
|
||||
names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
||||
skipinitialspace=True)
|
||||
predictions, labels, ids = [], [], []
|
||||
predictions_verb, predictions_noun = [], []
|
||||
discarded_labels, discarded_ids = [], []
|
||||
for segment in segment_list.iterrows():
|
||||
start_step = int(np.floor(segment[1]['start_f'] / 30 * cfg.DATA.FPS))
|
||||
vid = segment[1]['video'].strip()
|
||||
preds = (pred_scores[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
if cfg.MODEL.LSTR.V_N_CLASSIFIER:
|
||||
raise NotImplementedError(
|
||||
'For TeSTra with decomposed classifier (V+N), please use batch mode, ie ``MODEL.LSTR.INFERENCE_MODE batch``'
|
||||
)
|
||||
preds_v = (pred_scores_verb[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
preds_n = (pred_scores_noun[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
else:
|
||||
preds_v = None
|
||||
preds_n = None
|
||||
if start_step >= cfg.MODEL.LSTR.ANTICIPATION_LENGTH:
|
||||
predictions.append(preds)
|
||||
if preds_v is not None:
|
||||
predictions_verb.append(preds_v)
|
||||
if preds_n is not None:
|
||||
predictions_noun.append(preds_n)
|
||||
labels.append(segment[1][['verb', 'noun', 'action']].values.astype(int))
|
||||
ids.append(segment[1]['id'])
|
||||
else:
|
||||
discarded_labels.append(segment[1][['verb', 'noun', 'action']].values.astype(int))
|
||||
discarded_ids.append(segment[1]['id'])
|
||||
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'), index_col='id')
|
||||
|
||||
action_scores = np.concatenate(predictions)
|
||||
if len(predictions_verb) > 0:
|
||||
verb_scores = np.concatenate(predictions_verb)
|
||||
else:
|
||||
verb_scores = None
|
||||
if len(predictions_noun) > 0:
|
||||
noun_scores = np.concatenate(predictions_noun)
|
||||
else:
|
||||
noun_scores = None
|
||||
labels = np.array(labels)
|
||||
ids = np.array(ids)
|
||||
|
||||
vi = get_marginal_indexes(actions, 'verb')
|
||||
ni = get_marginal_indexes(actions, 'noun')
|
||||
|
||||
action_probs = softmax(action_scores.reshape(-1, action_scores.shape[-1]))
|
||||
if verb_scores is None:
|
||||
verb_scores = marginalize(action_probs, vi).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
else:
|
||||
verb_scores = softmax(verb_scores.reshape(-1, verb_scores.shape[-1]))
|
||||
verb_scores = verb_scores.reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
if noun_scores is None:
|
||||
noun_scores = marginalize(action_probs, ni).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
else:
|
||||
noun_scores = softmax(noun_scores.reshape(-1, noun_scores.shape[-1]))
|
||||
noun_scores = noun_scores.reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
|
||||
include_discarded = (cfg.DATA.DATA_NAME == 'EK100')
|
||||
if include_discarded:
|
||||
dlab = np.array(discarded_labels)
|
||||
dislab = np.array(discarded_ids)
|
||||
ids = np.concatenate([ids, dislab])
|
||||
num_disc = len(dlab)
|
||||
labels = np.concatenate([labels, dlab])
|
||||
verb_scores = np.concatenate((verb_scores, np.zeros((num_disc, *verb_scores.shape[1:]))))
|
||||
noun_scores = np.concatenate((noun_scores, np.zeros((num_disc, *noun_scores.shape[1:]))))
|
||||
action_scores = np.concatenate((action_scores, np.zeros((num_disc, *action_scores.shape[1:]))))
|
||||
|
||||
verb_labels, noun_labels, action_labels = labels[:, 0], labels[:, 1], labels[:, 2]
|
||||
verb_accuracies = topk_accuracy_multiple_timesteps(verb_scores, verb_labels)
|
||||
noun_accuracies = topk_accuracy_multiple_timesteps(noun_scores, noun_labels)
|
||||
action_accuracies = topk_accuracy_multiple_timesteps(action_scores, action_labels)
|
||||
|
||||
if cfg.DATA.DATA_NAME == 'EK55':
|
||||
many_shot_verbs = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_verbs.csv'))['verb_class'].values
|
||||
many_shot_nouns = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_nouns.csv'))['noun_class'].values
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'))
|
||||
a_to_vn = {a[1]['id']: tuple(a[1][['verb', 'noun']].values)
|
||||
for a in actions.iterrows()}
|
||||
many_shot_actions = [a for a, (v, n) in a_to_vn.items()
|
||||
if v in many_shot_verbs or n in many_shot_nouns]
|
||||
|
||||
verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels,
|
||||
k=5, classes=many_shot_verbs)
|
||||
noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels,
|
||||
k=5, classes=many_shot_nouns)
|
||||
action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels,
|
||||
k=5, classes=many_shot_actions)
|
||||
|
||||
all_accuracies = np.concatenate([verb_accuracies, noun_accuracies, action_accuracies,
|
||||
verb_recalls, noun_recalls, action_recalls])
|
||||
all_accuracies = all_accuracies[[0, 1, 6, 2, 3, 7, 4, 5, 8]]
|
||||
indices = [
|
||||
('Verb', 'Top-1 Accuracy'), ('Verb', 'Top-5 Accuracy'),
|
||||
('Verb', 'Mean Top-5 Recall'),
|
||||
('Noun', 'Top-1 Accuracy'), ('Noun', 'Top-5 Accuracy'),
|
||||
('Noun', 'Mean Top-5 Recall'),
|
||||
('Action', 'Top-1 Accuracy'), ('Action', 'Top-5 Accuracy'),
|
||||
('Action', 'Mean Top-5 Recall'),
|
||||
]
|
||||
else:
|
||||
overall_verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels, k=5)
|
||||
overall_noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels, k=5)
|
||||
overall_action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels, k=5)
|
||||
|
||||
unseen = pd.read_csv(osp.join(path_to_data, 'validation_unseen_participants_ids.csv'), names=['id'], squeeze=True)
|
||||
tail_verbs = pd.read_csv(osp.join(path_to_data, 'validation_tail_verbs_ids.csv'), names=['id'], squeeze=True)
|
||||
tail_nouns = pd.read_csv(osp.join(path_to_data, 'validation_tail_nouns_ids.csv'), names=['id'], squeeze=True)
|
||||
tail_actions = pd.read_csv(osp.join(path_to_data, 'validation_tail_actions_ids.csv'), names=['id'], squeeze=True)
|
||||
|
||||
unseen_bool_idx = pd.Series(ids).isin(unseen).values
|
||||
tail_verbs_bool_idx = pd.Series(ids).isin(tail_verbs).values
|
||||
tail_nouns_bool_idx = pd.Series(ids).isin(tail_nouns).values
|
||||
tail_actions_bool_idx = pd.Series(ids).isin(tail_actions).values
|
||||
|
||||
tail_verb_recalls = topk_recall_multiple_timesteps(
|
||||
verb_scores[tail_verbs_bool_idx], verb_labels[tail_verbs_bool_idx], k=5)
|
||||
tail_noun_recalls = topk_recall_multiple_timesteps(
|
||||
noun_scores[tail_nouns_bool_idx], noun_labels[tail_nouns_bool_idx], k=5)
|
||||
tail_action_recalls = topk_recall_multiple_timesteps(
|
||||
action_scores[tail_actions_bool_idx], action_labels[tail_actions_bool_idx], k=5)
|
||||
|
||||
unseen_verb_recalls = topk_recall_multiple_timesteps(
|
||||
verb_scores[unseen_bool_idx], verb_labels[unseen_bool_idx], k=5)
|
||||
unseen_noun_recalls = topk_recall_multiple_timesteps(
|
||||
noun_scores[unseen_bool_idx], noun_labels[unseen_bool_idx], k=5)
|
||||
unseen_action_recalls = topk_recall_multiple_timesteps(
|
||||
action_scores[unseen_bool_idx], action_labels[unseen_bool_idx], k=5)
|
||||
|
||||
all_accuracies = np.concatenate(
|
||||
[overall_verb_recalls, overall_noun_recalls, overall_action_recalls,
|
||||
unseen_verb_recalls, unseen_noun_recalls, unseen_action_recalls,
|
||||
tail_verb_recalls, tail_noun_recalls, tail_action_recalls]
|
||||
) #9 x 8
|
||||
|
||||
indices = [
|
||||
('Overall Mean Top-5 Recall', 'Verb'), ('Overall Mean Top-5 Recall', 'Noun'),
|
||||
('Overall Mean Top-5 Recall', 'Action'),
|
||||
('Unseen Mean Top-5 Recall', 'Verb'), ('Unseen Mean Top-5 Recall', 'Noun'),
|
||||
('Unseen Mean Top-5 Recall', 'Action'),
|
||||
('Tail Mean Top-5 Recall', 'Verb'), ('Tail Mean Top-5 Recall', 'Noun'),
|
||||
('Tail Mean Top-5 Recall', 'Action'),
|
||||
]
|
||||
|
||||
cc = np.linspace(1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_LENGTH,
|
||||
1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE,
|
||||
cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES, dtype=str)
|
||||
scores = pd.DataFrame(all_accuracies * 100, columns=cc,
|
||||
index=pd.MultiIndex.from_tuples(indices))
|
||||
logger.info(scores)
|
||||
|
||||
tta_verb = tta(verb_scores, verb_labels)
|
||||
tta_noun = tta(noun_scores, noun_labels)
|
||||
tta_action = tta(action_scores, action_labels)
|
||||
logger.info(f'\nMean TtA(5): VERB {tta_verb:0.2f} '
|
||||
f'NOUN: {tta_noun:0.2f} ACTION: {tta_action:0.2f}')
|
||||
26
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/lstr/lstr_trainer.py
vendored
Normal file
26
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/engines/lstr/lstr_trainer.py
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from ..base_trainers.perframe_det_trainer import do_perframe_det_train
|
||||
from ..engines import TRAINERS as registry
|
||||
|
||||
|
||||
@registry.register('LSTR')
|
||||
def do_lstr_train(cfg,
|
||||
data_loaders,
|
||||
model,
|
||||
criterion,
|
||||
optimizer,
|
||||
scheduler,
|
||||
device,
|
||||
checkpointer,
|
||||
logger):
|
||||
do_perframe_det_train(cfg,
|
||||
data_loaders,
|
||||
model,
|
||||
criterion,
|
||||
optimizer,
|
||||
scheduler,
|
||||
device,
|
||||
checkpointer,
|
||||
logger)
|
||||
4
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/evaluation/__init__.py
vendored
Normal file
4
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/evaluation/__init__.py
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from .evalution import compute_result
|
||||
42
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/evaluation/evalution.py
vendored
Normal file
42
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/evaluation/evalution.py
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from rekognition_online_action_detection.utils.registry import Registry
|
||||
|
||||
from .metrics import perframe_average_precision, perstage_average_precision
|
||||
from .postprocessing import postprocessing as default_pp
|
||||
|
||||
compute_result = Registry()
|
||||
|
||||
|
||||
@compute_result.register('perframe')
|
||||
def eval_perframe(cfg, ground_truth, prediction, **kwargs):
|
||||
class_names = kwargs.get('class_names', cfg.DATA.CLASS_NAMES)
|
||||
ignore_index = kwargs.get('ignore_index', cfg.DATA.IGNORE_INDEX)
|
||||
metrics = kwargs.get('metrics', cfg.DATA.METRICS)
|
||||
postprocessing = kwargs.get('postprocessing', default_pp(cfg.DATA.DATA_NAME))
|
||||
|
||||
return perframe_average_precision(
|
||||
ground_truth=ground_truth,
|
||||
prediction=prediction,
|
||||
class_names=class_names,
|
||||
ignore_index=ignore_index,
|
||||
metrics=metrics,
|
||||
postprocessing=postprocessing,
|
||||
)
|
||||
|
||||
|
||||
@compute_result.register('perstage')
|
||||
def eval_perstage(cfg, ground_truth, prediction, **kwargs):
|
||||
class_names = kwargs.get('class_names', cfg.DATA.CLASS_NAMES)
|
||||
ignore_index = kwargs.get('ignore_index', cfg.DATA.IGNORE_INDEX)
|
||||
metrics = kwargs.get('metrics', cfg.DATA.METRICS)
|
||||
postprocessing = kwargs.get('postprocessing', default_pp(cfg.DATA.DATA_NAME))
|
||||
return perstage_average_precision(
|
||||
ground_truth=ground_truth,
|
||||
prediction=prediction,
|
||||
class_names=class_names,
|
||||
ignore_index=ignore_index,
|
||||
metrics=metrics,
|
||||
postprocessing=postprocessing,
|
||||
)
|
||||
138
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/evaluation/metrics.py
vendored
Normal file
138
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/evaluation/metrics.py
vendored
Normal file
@@ -0,0 +1,138 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from collections import OrderedDict
|
||||
from multiprocessing import Pool
|
||||
|
||||
import numpy as np
|
||||
from sklearn.metrics import average_precision_score
|
||||
|
||||
|
||||
def calibrated_average_precision_score(y_true, y_score):
|
||||
"""Compute calibrated average precision (cAP), which is particularly
|
||||
proposed for the TVSeries dataset.
|
||||
"""
|
||||
y_true_sorted = y_true[np.argsort(-y_score)]
|
||||
tp = y_true_sorted.astype(float)
|
||||
fp = np.abs(y_true_sorted.astype(float) - 1)
|
||||
tps = np.cumsum(tp)
|
||||
fps = np.cumsum(fp)
|
||||
ratio = np.sum(tp == 0) / np.sum(tp)
|
||||
cprec = tps / (tps + fps / (ratio + np.finfo(float).eps) + np.finfo(float).eps)
|
||||
cap = np.sum(cprec[tp == 1]) / np.sum(tp)
|
||||
return cap
|
||||
|
||||
def perframe_average_precision(ground_truth,
|
||||
prediction,
|
||||
class_names,
|
||||
ignore_index,
|
||||
metrics,
|
||||
postprocessing):
|
||||
"""Compute (frame-level) average precision between ground truth and
|
||||
predictions data frames.
|
||||
"""
|
||||
result = OrderedDict()
|
||||
ground_truth = np.array(ground_truth)
|
||||
prediction = np.array(prediction)
|
||||
|
||||
# Postprocessing
|
||||
if postprocessing is not None:
|
||||
ground_truth, prediction = postprocessing(ground_truth, prediction)
|
||||
|
||||
# Build metrics
|
||||
if metrics == 'AP':
|
||||
compute_score = average_precision_score
|
||||
elif metrics == 'cAP':
|
||||
compute_score = calibrated_average_precision_score
|
||||
else:
|
||||
raise RuntimeError('Unknown metrics: {}'.format(metrics))
|
||||
|
||||
# set ignore classes including background
|
||||
ignore_set = set([ignore_index])
|
||||
# enable background according to ignore_index
|
||||
# ignore_set = set([ignore_index])
|
||||
|
||||
# Compute average precision
|
||||
result['per_class_AP'] = OrderedDict()
|
||||
|
||||
# when training, class_names becomes 2!
|
||||
for idx, class_name in enumerate(class_names):
|
||||
if idx not in ignore_set:
|
||||
if np.any(ground_truth[:, idx]):
|
||||
result['per_class_AP'][class_name] = compute_score(
|
||||
ground_truth[:, idx], prediction[:, idx])
|
||||
result['mean_AP'] = np.mean(list(result['per_class_AP'].values()))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_stage_pred_scores(gt_targets, pred_scores, perc_s, perc_e):
|
||||
starts = []
|
||||
ends = []
|
||||
stage_gt_targets = []
|
||||
stage_pred_scores = []
|
||||
for i in range(len(gt_targets)):
|
||||
if gt_targets[i] == 0:
|
||||
stage_gt_targets.append(gt_targets[i])
|
||||
stage_pred_scores.append(pred_scores[i])
|
||||
else:
|
||||
if i == 0 or gt_targets[i - 1] == 0:
|
||||
starts.append(i)
|
||||
if i == len(gt_targets) - 1 or gt_targets[i + 1] == 0:
|
||||
ends.append(i)
|
||||
if len(starts) != len(ends):
|
||||
raise ValueError('starts and ends cannot pair!')
|
||||
|
||||
action_lens = [ends[i] - starts[i] for i in range(len(starts))]
|
||||
stage_starts = [starts[i] + int(action_lens[i] * perc_s) for i in range(len(starts))]
|
||||
stage_ends = [max(stage_starts[i] + 1, starts[i] + int(action_lens[i] * perc_e)) for i in range(len(starts))]
|
||||
for i in range(len(starts)):
|
||||
stage_gt_targets.extend(gt_targets[stage_starts[i]: stage_ends[i]])
|
||||
stage_pred_scores.extend(pred_scores[stage_starts[i]: stage_ends[i]])
|
||||
return np.array(stage_gt_targets), np.array(stage_pred_scores)
|
||||
|
||||
|
||||
def perstage_average_precision(ground_truth,
|
||||
prediction,
|
||||
class_names,
|
||||
ignore_index,
|
||||
metrics,
|
||||
postprocessing):
|
||||
result = OrderedDict()
|
||||
ground_truth = np.array(ground_truth)
|
||||
prediction = np.array(prediction)
|
||||
|
||||
# Postprocessing
|
||||
if postprocessing is not None:
|
||||
ground_truth, prediction = postprocessing(ground_truth, prediction)
|
||||
|
||||
# Build metrics
|
||||
if metrics == 'AP':
|
||||
compute_score = average_precision_score
|
||||
elif metrics == 'cAP':
|
||||
compute_score = calibrated_average_precision_score
|
||||
else:
|
||||
raise RuntimeError('Unknown metrics: {}'.format(metrics))
|
||||
|
||||
# Ignore backgroud class
|
||||
ignore_set = set([0, ignore_index])
|
||||
|
||||
# Compute average precision
|
||||
for perc_s in range(10):
|
||||
perc_e = perc_s + 1
|
||||
stage_name = '{:2}%_{:3}%'.format(perc_s * 10, perc_e * 10)
|
||||
result[stage_name] = OrderedDict({'per_class_AP': OrderedDict()})
|
||||
for idx, class_name in enumerate(class_names):
|
||||
if idx not in ignore_set:
|
||||
stage_gt_targets, stage_pred_scores = get_stage_pred_scores(
|
||||
(ground_truth[:, idx] == 1).astype(int),
|
||||
prediction[:, idx],
|
||||
perc_s / 10,
|
||||
perc_e / 10,
|
||||
)
|
||||
result[stage_name]['per_class_AP'][class_name] = \
|
||||
compute_score(stage_gt_targets, stage_pred_scores)
|
||||
result[stage_name]['mean_AP'] = \
|
||||
np.mean(list(result[stage_name]['per_class_AP'].values()))
|
||||
|
||||
return result
|
||||
36
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/evaluation/postprocessing.py
vendored
Normal file
36
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/evaluation/postprocessing.py
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def postprocessing(data_name):
|
||||
|
||||
def thumos_postprocessing(ground_truth, prediction, smooth=False, switch=False):
|
||||
"""
|
||||
We follow (Shou et al., 2017) and adopt their perframe postprocessing method on THUMOS'14 datset.
|
||||
Source: https://bitbucket.org/columbiadvmm/cdc/src/master/THUMOS14/eval/PreFrameLabeling/compute_framelevel_mAP.m
|
||||
"""
|
||||
|
||||
# Simple temporal smoothing via NMS of 5-frames window
|
||||
if smooth:
|
||||
prob = np.copy(prediction)
|
||||
prob1 = prob.reshape(1, prob.shape[0], prob.shape[1])
|
||||
prob2 = np.append(prob[0, :].reshape(1, -1), prob[0: -1, :], axis=0).reshape(1, prob.shape[0], prob.shape[1])
|
||||
prob3 = np.append(prob[1:, :], prob[-1, :].reshape(1, -1), axis=0).reshape(1, prob.shape[0], prob.shape[1])
|
||||
prob4 = np.append(prob[0: 2, :], prob[0: -2, :], axis=0).reshape(1, prob.shape[0], prob.shape[1])
|
||||
prob5 = np.append(prob[2:, :], prob[-2:, :], axis=0).reshape(1, prob.shape[0], prob.shape[1])
|
||||
probsmooth = np.squeeze(np.max(np.concatenate((prob1, prob2, prob3, prob4, prob5), axis=0), axis=0))
|
||||
prediction = np.copy(probsmooth)
|
||||
|
||||
# Assign cliff diving (5) as diving (8)
|
||||
if switch:
|
||||
switch_index = np.where(prediction[:, 5] > prediction[:, 8])[0]
|
||||
prediction[switch_index, 8] = prediction[switch_index, 5]
|
||||
|
||||
# Remove ambiguous (21)
|
||||
valid_index = np.where(ground_truth[:, 21] != 1)[0]
|
||||
|
||||
return ground_truth[valid_index], prediction[valid_index]
|
||||
|
||||
return {'THUMOS': thumos_postprocessing}.get(data_name, None)
|
||||
5
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/__init__.py
vendored
Normal file
5
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/__init__.py
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from .lstr import *
|
||||
from .models import build_model
|
||||
95
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/feature_head.py
vendored
Normal file
95
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/feature_head.py
vendored
Normal file
@@ -0,0 +1,95 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
__all__ = ['build_feature_head']
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from rekognition_online_action_detection.config.feature_sizes import \
|
||||
FEATURE_SIZES
|
||||
from rekognition_online_action_detection.utils.registry import Registry
|
||||
|
||||
FEATURE_HEADS = Registry()
|
||||
|
||||
@FEATURE_HEADS.register('EasyCom')
|
||||
@FEATURE_HEADS.register('EGO4D')
|
||||
@FEATURE_HEADS.register('EGO4DVAL')
|
||||
@FEATURE_HEADS.register('THUMOS')
|
||||
@FEATURE_HEADS.register('TVSeries')
|
||||
@FEATURE_HEADS.register('EK55')
|
||||
@FEATURE_HEADS.register('EK100')
|
||||
@FEATURE_HEADS.register('YTConv')
|
||||
class BaseFeatureHead(nn.Module):
|
||||
def __init__(self, cfg):
|
||||
super(BaseFeatureHead, self).__init__()
|
||||
|
||||
if cfg.INPUT.MODALITY in ['visual', 'motion', 'object',
|
||||
'visual+motion', 'motion+object', 'visual+object',
|
||||
'visual+motion+object']:
|
||||
self.with_visual = 'visual' in cfg.INPUT.MODALITY
|
||||
self.with_motion = 'motion' in cfg.INPUT.MODALITY
|
||||
self.with_object = 'object' in cfg.INPUT.MODALITY
|
||||
else:
|
||||
raise RuntimeError('Unknown modality of {}'.format(cfg.INPUT.MODALITY))
|
||||
|
||||
INPUT_VISUAL_FEATURE = cfg.INPUT.VISUAL_FEATURE
|
||||
INPUT_MOTION_FEATURE = cfg.INPUT.MOTION_FEATURE
|
||||
INPUT_OBJECT_FEATURE = cfg.INPUT.OBJECT_FEATURE
|
||||
|
||||
if self.with_visual and self.with_motion and self.with_object:
|
||||
visual_size = FEATURE_SIZES[INPUT_VISUAL_FEATURE]
|
||||
motion_size = FEATURE_SIZES[INPUT_MOTION_FEATURE]
|
||||
object_size = FEATURE_SIZES[INPUT_OBJECT_FEATURE]
|
||||
fusion_size = visual_size + motion_size + object_size
|
||||
elif self.with_visual and self.with_motion:
|
||||
visual_size = FEATURE_SIZES[INPUT_VISUAL_FEATURE]
|
||||
motion_size = FEATURE_SIZES[INPUT_MOTION_FEATURE]
|
||||
fusion_size = visual_size + motion_size
|
||||
elif self.with_visual:
|
||||
fusion_size = FEATURE_SIZES[INPUT_VISUAL_FEATURE]
|
||||
elif self.with_motion:
|
||||
fusion_size = FEATURE_SIZES[INPUT_MOTION_FEATURE]
|
||||
elif self.with_object:
|
||||
fusion_size = FEATURE_SIZES[INPUT_OBJECT_FEATURE]
|
||||
|
||||
self.fusion_size = fusion_size
|
||||
self.d_model = fusion_size
|
||||
|
||||
if cfg.MODEL.FEATURE_HEAD.LINEAR_ENABLED:
|
||||
if cfg.MODEL.FEATURE_HEAD.LINEAR_OUT_FEATURES != -1:
|
||||
self.d_model = cfg.MODEL.FEATURE_HEAD.LINEAR_OUT_FEATURES
|
||||
self.input_linear = nn.Sequential(
|
||||
nn.Linear(fusion_size, self.d_model),
|
||||
nn.ReLU(inplace=True),
|
||||
)
|
||||
else:
|
||||
self.input_linear = nn.Identity()
|
||||
if cfg.MODEL.FEATURE_HEAD.DROPOUT > 0:
|
||||
self.dropout = nn.Dropout(cfg.MODEL.FEATURE_HEAD.DROPOUT)
|
||||
else:
|
||||
self.dropout = None
|
||||
|
||||
def forward(self, visual_input, motion_input, object_input):
|
||||
if self.with_visual and self.with_motion and self.with_object:
|
||||
fusion_input = torch.cat((visual_input, motion_input,
|
||||
object_input), dim=-1)
|
||||
elif self.with_visual and self.with_motion:
|
||||
fusion_input = torch.cat((visual_input, motion_input), dim=-1)
|
||||
elif self.with_visual:
|
||||
fusion_input = visual_input
|
||||
elif self.with_motion:
|
||||
fusion_input = motion_input
|
||||
elif self.with_object:
|
||||
fusion_input = object_input
|
||||
if self.dropout is not None:
|
||||
fusion_input = self.dropout(fusion_input)
|
||||
fusion_input = self.input_linear(fusion_input)
|
||||
return fusion_input
|
||||
|
||||
|
||||
def build_feature_head(cfg):
|
||||
data_names = ['YTConv', 'EGO4D']
|
||||
data_name = next((name for name in data_names if name in cfg.DATA.DATA_NAME), cfg.DATA.DATA_NAME)
|
||||
|
||||
feature_head = FEATURE_HEADS[data_name]
|
||||
return feature_head(cfg)
|
||||
424
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/lstr.py
vendored
Normal file
424
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/lstr.py
vendored
Normal file
@@ -0,0 +1,424 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from ..utils.ek_utils import action_to_noun_map, action_to_verb_map
|
||||
from . import transformer as tr
|
||||
from .feature_head import build_feature_head
|
||||
from .models import META_ARCHITECTURES as registry
|
||||
from .normalized_linear import NormalizedLinear
|
||||
|
||||
|
||||
class LSTR(nn.Module):
|
||||
|
||||
def __init__(self, cfg):
|
||||
super(LSTR, self).__init__()
|
||||
|
||||
# Build long feature heads
|
||||
self.long_memory_num_samples = cfg.MODEL.LSTR.LONG_MEMORY_NUM_SAMPLES
|
||||
self.long_enabled = self.long_memory_num_samples > 0
|
||||
if self.long_enabled:
|
||||
self.feature_head_long = build_feature_head(cfg)
|
||||
|
||||
# Build work feature head
|
||||
self.work_memory_num_samples = cfg.MODEL.LSTR.WORK_MEMORY_NUM_SAMPLES
|
||||
self.work_enabled = self.work_memory_num_samples > 0
|
||||
if self.work_enabled:
|
||||
self.feature_head_work = build_feature_head(cfg)
|
||||
|
||||
self.d_model = self.feature_head_work.d_model
|
||||
self.num_heads = cfg.MODEL.LSTR.NUM_HEADS
|
||||
self.dim_feedforward = cfg.MODEL.LSTR.DIM_FEEDFORWARD
|
||||
self.dropout = cfg.MODEL.LSTR.DROPOUT
|
||||
self.activation = cfg.MODEL.LSTR.ACTIVATION
|
||||
self.num_classes = cfg.DATA.NUM_CLASSES
|
||||
|
||||
self.use_v_n = cfg.MODEL.LSTR.V_N_CLASSIFIER
|
||||
|
||||
self.encoder_attention_type = cfg.MODEL.LSTR.ENC_ATTENTION_TYPE
|
||||
self.decay_alpha = cfg.MODEL.LSTR.ENC_ATTENTION_DECAY
|
||||
|
||||
self.anticipation_num_samples = cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES
|
||||
|
||||
# Build position encoding
|
||||
self.pos_encoding = tr.PositionalEncoding(self.d_model, self.dropout)
|
||||
|
||||
self.long_memory_use_pe = cfg.MODEL.LSTR.LONG_MEMORY_USE_PE
|
||||
self.work_memory_use_pe = cfg.MODEL.LSTR.WORK_MEMORY_USE_PE
|
||||
self.include_work = cfg.MODEL.LSTR.LONG_MEMORY_INCLUDE_WORK
|
||||
self.include_work2 = cfg.MODEL.LSTR.LONG_MEMORY_INCLUDE_WORK2
|
||||
if self.include_work and self.include_work2:
|
||||
raise ValueError('cfg.MODEL.LSTR.LONG_MEMORY_INCLUDE_WORK and'
|
||||
'cfg.MODEL.LSTR.LONG_MEMORY_INCLUDE_WORK2 should'
|
||||
'not be both True')
|
||||
|
||||
# Build LSTR encoder
|
||||
if self.long_enabled:
|
||||
self.enc_queries = nn.ModuleList()
|
||||
self.enc_modules = nn.ModuleList()
|
||||
for i, param in enumerate(cfg.MODEL.LSTR.ENC_MODULE):
|
||||
if param[0] != -1:
|
||||
self.enc_queries.append(nn.Embedding(param[0], self.d_model))
|
||||
# in TeSTra, not like LSTR, gets atteyntion_type, decay_alpha as parameter
|
||||
# however, the parameter does not change anything from LSTR
|
||||
enc_layer = tr.TransformerDecoderLayer(
|
||||
self.d_model, self.num_heads, self.dim_feedforward,
|
||||
self.dropout, self.activation,
|
||||
attention_type=self.encoder_attention_type if i == 0 else 'dotproduct',
|
||||
decay_alpha=self.decay_alpha if i == 0 else 1.0)
|
||||
self.enc_modules.append(tr.TransformerDecoder(
|
||||
enc_layer, param[1], tr.layer_norm(self.d_model, param[2])))
|
||||
else:
|
||||
self.enc_queries.append(None)
|
||||
enc_layer = tr.TransformerEncoderLayer(
|
||||
self.d_model, self.num_heads, self.dim_feedforward,
|
||||
self.dropout, self.activation,
|
||||
attention_type=self.encoder_attention_type)
|
||||
self.enc_modules.append(tr.TransformerEncoder(
|
||||
enc_layer, param[1], tr.layer_norm(self.d_model, param[2])))
|
||||
else:
|
||||
self.register_parameter('enc_queries', None)
|
||||
self.register_parameter('enc_modules', None)
|
||||
|
||||
# Build LSTR decoder
|
||||
if self.long_enabled:
|
||||
param = cfg.MODEL.LSTR.DEC_MODULE
|
||||
dec_layer = tr.TransformerDecoderLayer(
|
||||
self.d_model, self.num_heads, self.dim_feedforward,
|
||||
self.dropout, self.activation)
|
||||
self.dec_modules = tr.TransformerDecoder(
|
||||
dec_layer, param[1], tr.layer_norm(self.d_model, param[2]))
|
||||
else:
|
||||
param = cfg.MODEL.LSTR.DEC_MODULE
|
||||
dec_layer = tr.TransformerEncoderLayer(
|
||||
self.d_model, self.num_heads, self.dim_feedforward,
|
||||
self.dropout, self.activation)
|
||||
self.dec_modules = tr.TransformerEncoder(
|
||||
dec_layer, param[1], tr.layer_norm(self.d_model, param[2]))
|
||||
|
||||
# Build decode token (for anticipation)
|
||||
if self.anticipation_num_samples > 0:
|
||||
self.dec_query = nn.Embedding(self.anticipation_num_samples, self.d_model)
|
||||
else:
|
||||
self.register_parameter('dec_query', None)
|
||||
|
||||
# Build classifier
|
||||
if cfg.MODEL.LSTR.DROPOUT_CLS > 0:
|
||||
self.dropout_cls = nn.Dropout(cfg.MODEL.LSTR.DROPOUT_CLS)
|
||||
if self.use_v_n:
|
||||
self.dropout_cls_v = nn.Dropout(cfg.MODEL.LSTR.DROPOUT_CLS)
|
||||
self.dropout_cls_n = nn.Dropout(cfg.MODEL.LSTR.DROPOUT_CLS)
|
||||
else:
|
||||
self.dropout_cls = None
|
||||
if cfg.MODEL.LSTR.FC_NORM:
|
||||
self.classifier = NormalizedLinear(self.d_model, self.num_classes)
|
||||
else:
|
||||
self.classifier = nn.Linear(self.d_model, self.num_classes)
|
||||
|
||||
if self.use_v_n:
|
||||
a_to_v = action_to_verb_map(cfg.DATA.EK_EXT_PATH,
|
||||
action_offset=True,
|
||||
verb_offset=True)
|
||||
a_to_n = action_to_noun_map(cfg.DATA.EK_EXT_PATH,
|
||||
action_offset=True,
|
||||
noun_offset=True)
|
||||
num_verbs = max(set(a_to_v.values())) + 1
|
||||
num_nouns = max(set(a_to_n.values())) + 1
|
||||
print('Number of verbs (inc. 0):', num_verbs)
|
||||
print('Number of nouns (inc. 0):', num_nouns)
|
||||
self.classifier_verb = nn.Linear(self.d_model, num_verbs)
|
||||
self.classifier_noun = nn.Linear(self.d_model, num_nouns)
|
||||
|
||||
self.pred_future = 'PRED_FUTURE' in list(zip(*cfg.MODEL.CRITERIONS))[0]
|
||||
|
||||
def forward(self, visual_inputs, motion_inputs, object_inputs, memory_key_padding_mask=None):
|
||||
if self.long_enabled:
|
||||
# Compute long memories
|
||||
if self.long_memory_use_pe:
|
||||
long_memories = self.pos_encoding(self.feature_head_long(
|
||||
visual_inputs[:, :self.long_memory_num_samples if not self.include_work2 else None],
|
||||
motion_inputs[:, :self.long_memory_num_samples if not self.include_work2 else None],
|
||||
object_inputs[:, :self.long_memory_num_samples if not self.include_work2 else None],
|
||||
).transpose(0, 1))
|
||||
long_memories = long_memories.transpose(0, 1) ## transpose back
|
||||
else:
|
||||
long_memories = self.feature_head_long(
|
||||
visual_inputs[:, :self.long_memory_num_samples if not self.include_work2 else None],
|
||||
motion_inputs[:, :self.long_memory_num_samples if not self.include_work2 else None],
|
||||
object_inputs[:, :self.long_memory_num_samples if not self.include_work2 else None],
|
||||
)
|
||||
batch_size = long_memories.shape[0]
|
||||
# if long_memories.ndim == 5:
|
||||
# long_memories = self.pos_encoding_3d_spatial(long_memories)
|
||||
long_memories = long_memories.view(batch_size, -1, self.d_model).transpose(0, 1)
|
||||
# print(long_memories.shape) # T(*H*W), B, C
|
||||
|
||||
if len(self.enc_modules) > 0:
|
||||
enc_queries = [
|
||||
enc_query.weight.unsqueeze(1).repeat(1, long_memories.shape[1], 1)
|
||||
if enc_query is not None else None
|
||||
for enc_query in self.enc_queries
|
||||
]
|
||||
|
||||
if self.include_work2:
|
||||
memory_key_padding_mask = F.pad(memory_key_padding_mask,
|
||||
(0, self.work_memory_num_samples),
|
||||
'constant', 0)
|
||||
# Encode long memories
|
||||
if enc_queries[0] is not None:
|
||||
long_memories = self.enc_modules[0](enc_queries[0], long_memories,
|
||||
memory_key_padding_mask=memory_key_padding_mask)
|
||||
else:
|
||||
long_memories = self.enc_modules[0](long_memories)
|
||||
for enc_query, enc_module in zip(enc_queries[1:], self.enc_modules[1:]):
|
||||
if enc_query is not None:
|
||||
long_memories = enc_module(enc_query, long_memories)
|
||||
else:
|
||||
long_memories = enc_module(long_memories)
|
||||
|
||||
# Concatenate memories
|
||||
if self.long_enabled:
|
||||
memory = long_memories
|
||||
|
||||
if self.work_enabled:
|
||||
# Compute work memories
|
||||
if visual_inputs.ndim == 5:
|
||||
visual_inputs_avg = visual_inputs[:, self.long_memory_num_samples:].mean((2, 3))
|
||||
motion_inputs_avg = motion_inputs[:, self.long_memory_num_samples:].mean((2, 3))
|
||||
object_inputs_avg = object_inputs[:, self.long_memory_num_samples:].mean((2, 3))
|
||||
else:
|
||||
visual_inputs_avg = visual_inputs[:, self.long_memory_num_samples:]
|
||||
motion_inputs_avg = motion_inputs[:, self.long_memory_num_samples:]
|
||||
object_inputs_avg = object_inputs[:, self.long_memory_num_samples:]
|
||||
work_memories_no_pe = self.feature_head_work(
|
||||
visual_inputs_avg,
|
||||
motion_inputs_avg,
|
||||
object_inputs_avg
|
||||
# object_inputs[:, self.long_memory_num_samples:],
|
||||
).transpose(0, 1)
|
||||
if self.work_memory_use_pe:
|
||||
work_memories = self.pos_encoding(work_memories_no_pe,
|
||||
padding=self.long_memory_num_samples if self.long_memory_use_pe else 0)
|
||||
else:
|
||||
work_memories = work_memories_no_pe
|
||||
if self.dec_query is not None:
|
||||
anticipate_memories = self.pos_encoding(
|
||||
self.dec_query.weight.unsqueeze(1).repeat(1, work_memories.shape[1], 1),
|
||||
padding=(self.long_memory_num_samples + self.work_memory_num_samples
|
||||
if self.long_memory_use_pe else self.work_memory_num_samples))
|
||||
work_memories = torch.cat((work_memories, anticipate_memories), dim=0)
|
||||
# work_memories = self.feature_head_work(
|
||||
# visual_inputs[:, self.long_memory_num_samples:],
|
||||
# motion_inputs[:, self.long_memory_num_samples:],
|
||||
# ).transpose(0, 1)
|
||||
|
||||
|
||||
# Build mask
|
||||
mask = tr.generate_square_subsequent_mask(
|
||||
work_memories.shape[0])
|
||||
mask = mask.to(work_memories.device)
|
||||
# Compute output
|
||||
if self.long_enabled:
|
||||
if self.include_work:
|
||||
extended_memory = torch.cat((memory, work_memories), dim=0)
|
||||
extended_mask = F.pad(mask,
|
||||
(memory.shape[0], 0),
|
||||
'constant', 0)
|
||||
extended_mask = extended_mask.to(extended_memory.device)
|
||||
output = self.dec_modules(
|
||||
work_memories,
|
||||
memory=extended_memory,
|
||||
tgt_mask=mask,
|
||||
memory_mask=extended_mask,
|
||||
)
|
||||
else:
|
||||
output = self.dec_modules(
|
||||
work_memories,
|
||||
memory=memory,
|
||||
tgt_mask=mask,
|
||||
)
|
||||
else:
|
||||
output = self.dec_modules(
|
||||
work_memories,
|
||||
src_mask=mask,
|
||||
)
|
||||
|
||||
full_orig_feats = self.feature_head_work(
|
||||
visual_inputs_avg,
|
||||
motion_inputs_avg,
|
||||
object_inputs_avg
|
||||
# object_inputs[:, self.long_memory_num_samples:]
|
||||
).transpose(0, 1)
|
||||
all_outputs_decoded = output
|
||||
|
||||
# Compute classification score
|
||||
if self.dropout_cls is not None:
|
||||
output_a = self.dropout_cls(output)
|
||||
else:
|
||||
output_a = output
|
||||
|
||||
# output_a
|
||||
# [workmem_len + ant_len, batch_size, self.d_model]: [50, 16, 1024]
|
||||
# score (before transpose) [len, batch_size, num_classes]
|
||||
# self.classifier: nn.Linear(self.d_model, self.num_classes)
|
||||
score = self.classifier(output_a)
|
||||
score = score.transpose(0, 1)
|
||||
|
||||
if self.use_v_n:
|
||||
output_v = self.dropout_cls_v(output)
|
||||
output_n = self.dropout_cls_n(output)
|
||||
score_verb = self.classifier_verb(output_v).transpose(0, 1)
|
||||
score_noun = self.classifier_noun(output_n).transpose(0, 1)
|
||||
score = (score, score_verb, score_noun)
|
||||
if self.pred_future:
|
||||
return (score,
|
||||
all_outputs_decoded[1: self.work_memory_num_samples + 1, ...],
|
||||
full_orig_feats)
|
||||
else:
|
||||
return score
|
||||
|
||||
|
||||
@registry.register('LSTR')
|
||||
class LSTRStream(LSTR):
|
||||
|
||||
def __init__(self, cfg):
|
||||
super(LSTRStream, self).__init__(cfg)
|
||||
|
||||
############################
|
||||
# Cache for stream inference
|
||||
############################
|
||||
self.long_memories_cache = None
|
||||
self.compressed_long_memories_cache = None
|
||||
|
||||
def stream_inference(self,
|
||||
long_visual_inputs,
|
||||
long_motion_inputs,
|
||||
long_object_inputs,
|
||||
work_visual_inputs,
|
||||
work_motion_inputs,
|
||||
work_object_inputs,
|
||||
memory_key_padding_mask=None,
|
||||
cache_num=1,
|
||||
cache_id=0):
|
||||
assert self.long_enabled, 'Long-term memory cannot be empty for stream inference'
|
||||
assert len(self.enc_modules) > 0, 'LSTR encoder cannot be disabled for stream inference'
|
||||
|
||||
if ((long_visual_inputs is not None) and
|
||||
(long_motion_inputs is not None) and
|
||||
(long_object_inputs is not None)):
|
||||
# Compute long memories
|
||||
long_memories = self.feature_head_long(
|
||||
long_visual_inputs if not self.include_work2 else torch.cat((long_visual_inputs, work_visual_inputs), dim=1),
|
||||
long_motion_inputs if not self.include_work2 else torch.cat((long_motion_inputs, work_motion_inputs), dim=1),
|
||||
long_object_inputs if not self.include_work2 else torch.cat((long_object_inputs, work_object_inputs), dim=1),
|
||||
).transpose(0, 1)
|
||||
|
||||
if self.long_memories_cache is None:
|
||||
self.long_memories_cache = [long_memories for _ in range(cache_num)]
|
||||
else:
|
||||
self.long_memories_cache[cache_id] = torch.cat((
|
||||
self.long_memories_cache[cache_id][1:], long_memories
|
||||
))
|
||||
|
||||
long_memories = self.long_memories_cache[cache_id]
|
||||
pos = self.pos_encoding.pe[:self.long_memory_num_samples, :]
|
||||
|
||||
enc_queries = [
|
||||
enc_query.weight.unsqueeze(1).repeat(1, long_memories.shape[1], 1)
|
||||
if enc_query is not None else None
|
||||
for enc_query in self.enc_queries
|
||||
]
|
||||
|
||||
if self.include_work2:
|
||||
memory_key_padding_mask = F.pad(memory_key_padding_mask,
|
||||
(0, self.work_memory_num_samples),
|
||||
'constant', 0)
|
||||
# Encode long memories
|
||||
long_memories = self.enc_modules[0].stream_inference(enc_queries[0], long_memories, pos,
|
||||
memory_key_padding_mask=memory_key_padding_mask,
|
||||
cache_num=cache_num, cache_id=cache_id)
|
||||
self.compressed_long_memories_cache = long_memories
|
||||
for enc_query, enc_module in zip(enc_queries[1:], self.enc_modules[1:]):
|
||||
if enc_query is not None:
|
||||
long_memories = enc_module(enc_query, long_memories)
|
||||
else:
|
||||
long_memories = enc_module(long_memories)
|
||||
else:
|
||||
long_memories = self.compressed_long_memories_cache
|
||||
|
||||
enc_queries = [
|
||||
enc_query.weight.unsqueeze(1).repeat(1, long_memories.shape[1], 1)
|
||||
if enc_query is not None else None
|
||||
for enc_query in self.enc_queries
|
||||
]
|
||||
|
||||
# Encode long memories
|
||||
for enc_query, enc_module in zip(enc_queries[1:], self.enc_modules[1:]):
|
||||
if enc_query is not None:
|
||||
long_memories = enc_module(enc_query, long_memories)
|
||||
else:
|
||||
long_memories = enc_module(long_memories)
|
||||
|
||||
# Concatenate memories
|
||||
if self.long_enabled:
|
||||
memory = long_memories
|
||||
|
||||
if self.work_enabled:
|
||||
# Compute work memories
|
||||
work_memories = self.pos_encoding(self.feature_head_work(
|
||||
work_visual_inputs,
|
||||
work_motion_inputs,
|
||||
work_object_inputs,
|
||||
).transpose(0, 1),
|
||||
padding=self.long_memory_num_samples if self.long_memory_use_pe else 0)
|
||||
if self.dec_query is not None:
|
||||
anticipate_memories = self.pos_encoding(
|
||||
self.dec_query.weight.unsqueeze(1).repeat(1, work_memories.shape[1], 1),
|
||||
padding=(self.long_memory_num_samples + self.work_memory_num_samples
|
||||
if self.long_memory_use_pe else self.work_memory_num_samples))
|
||||
work_memories = torch.cat((work_memories, anticipate_memories))
|
||||
|
||||
# Build mask
|
||||
mask = tr.generate_square_subsequent_mask(
|
||||
work_memories.shape[0])
|
||||
mask = mask.to(work_memories.device)
|
||||
|
||||
# Compute output
|
||||
if self.long_enabled:
|
||||
if self.include_work:
|
||||
extended_memory = torch.cat((memory, work_memories), dim=0)
|
||||
extended_mask = F.pad(mask,
|
||||
(memory.shape[0], 0),
|
||||
'constant', 0)
|
||||
extended_mask = extended_mask.to(extended_memory.device)
|
||||
output = self.dec_modules(
|
||||
work_memories,
|
||||
memory=extended_memory,
|
||||
tgt_mask=mask,
|
||||
memory_mask=extended_mask,
|
||||
)
|
||||
else:
|
||||
output = self.dec_modules(
|
||||
work_memories,
|
||||
memory=memory,
|
||||
tgt_mask=mask,
|
||||
)
|
||||
else:
|
||||
output = self.dec_modules(
|
||||
work_memories,
|
||||
src_mask=mask,
|
||||
)
|
||||
|
||||
# Compute classification score
|
||||
score = self.classifier(output)
|
||||
import pdb; pdb.set_trace()
|
||||
|
||||
return score.transpose(0, 1)
|
||||
|
||||
def clear_cache(self):
|
||||
self.long_memories_cache = None
|
||||
self.compressed_long_memories_cache = None
|
||||
self.enc_modules[0].clear_cache()
|
||||
15
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/models.py
vendored
Normal file
15
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/models.py
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
__all__ = ['build_model']
|
||||
|
||||
from rekognition_online_action_detection.utils.registry import Registry
|
||||
|
||||
META_ARCHITECTURES = Registry()
|
||||
|
||||
def build_model(cfg, device=None):
|
||||
model = META_ARCHITECTURES[cfg.MODEL.MODEL_NAME](cfg)
|
||||
from .weights_init import weights_init
|
||||
|
||||
model.apply(weights_init)
|
||||
return model.to(device)
|
||||
57
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/normalized_linear.py
vendored
Normal file
57
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/normalized_linear.py
vendored
Normal file
@@ -0,0 +1,57 @@
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch import Tensor
|
||||
from torch.nn.parameter import Parameter
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class NormalizedLinear(nn.Module):
|
||||
r"""Normalized linear layer which was proposed by Jiaqi Wang et. al. in
|
||||
their technical report for LVIS workshop at ECCV 2020.
|
||||
https://arxiv.org/pdf/2008.10032.pdf
|
||||
Args:
|
||||
in_features: size of each input sample
|
||||
out_features: size of each output sample
|
||||
bias: If set to ``False``, the layer will not learn an additive bias.
|
||||
Default: ``True``
|
||||
tau: a temperature factor, set as 20 by default by following the paper.
|
||||
"""
|
||||
__constants__ = ['in_features', 'out_features']
|
||||
in_features: int
|
||||
out_features: int
|
||||
weight: Tensor
|
||||
|
||||
def __init__(self, in_features: int, out_features: int, bias: bool = True, tau: float = 20) -> None:
|
||||
super().__init__()
|
||||
self.in_features = in_features
|
||||
self.out_features = out_features
|
||||
self.weight = Parameter(torch.Tensor(out_features, in_features))
|
||||
if bias:
|
||||
self.bias = Parameter(torch.Tensor(out_features))
|
||||
else:
|
||||
self.register_parameter('bias', None)
|
||||
self.reset_parameters()
|
||||
self.tau = tau
|
||||
|
||||
def reset_parameters(self) -> None:
|
||||
nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
|
||||
if self.bias is not None:
|
||||
fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight)
|
||||
bound = 1 / math.sqrt(fan_in)
|
||||
nn.init.uniform_(self.bias, -bound, bound)
|
||||
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
|
||||
return F.linear(
|
||||
input / torch.norm(input, dim=-1, keepdim=True),
|
||||
self.tau * self.weight /
|
||||
torch.norm(self.weight, dim=-1, keepdim=True),
|
||||
self.bias
|
||||
)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return 'in_features={}, out_features={}, bias={}'.format(
|
||||
self.in_features, self.out_features, self.bias is not None
|
||||
)
|
||||
10
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/transformer/__init__.py
vendored
Normal file
10
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/transformer/__init__.py
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from .position_encoding import PositionalEncoding
|
||||
from .position_encoding import PositionalEncoding2D
|
||||
from .transformer import MultiheadAttention
|
||||
from .transformer import Transformer
|
||||
from .transformer import TransformerEncoder, TransformerEncoderLayer
|
||||
from .transformer import TransformerDecoder, TransformerDecoderLayer
|
||||
from .utils import layer_norm, generate_square_subsequent_mask
|
||||
@@ -0,0 +1,456 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import torch
|
||||
from torch._C import device
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
import time
|
||||
|
||||
class DotProductAttention(nn.Module):
|
||||
|
||||
def __init__(self, dropout=0.0, alpha=1.0, num_heads=8):
|
||||
super(DotProductAttention, self).__init__()
|
||||
|
||||
self.dropout = dropout
|
||||
self.alpha = alpha
|
||||
self.num_heads = num_heads
|
||||
self.decay_mat = None
|
||||
|
||||
def forward(self, q, k, v, attn_mask=None):
|
||||
attn_output_weights = torch.bmm(q, k.transpose(1, 2))
|
||||
|
||||
if attn_mask is not None:
|
||||
attn_output_weights += attn_mask
|
||||
|
||||
# attn_output_weights = F.softmax(attn_output_weights, dim=-1)
|
||||
# attn_output_weights = F.dropout(attn_output_weights,
|
||||
# p=self.dropout,
|
||||
# training=self.training)
|
||||
if self.alpha != 1.0 and self.decay_mat is None:
|
||||
self.decay_mat = torch.vander(torch.tensor([self.alpha] * self.num_heads), N=v.shape[1])
|
||||
self.decay_mat = self.decay_mat.to(v.device)
|
||||
if self.alpha != 1.0:
|
||||
bs = attn_output_weights.shape[0] // self.num_heads
|
||||
decay_mat = self.decay_mat.unsqueeze(0).repeat(bs, 1, 1)
|
||||
decay_mat = decay_mat.view(-1, *decay_mat.shape[2:])
|
||||
attn_output_weights = torch.log(decay_mat[:, None, :] + 1e-10) + attn_output_weights
|
||||
attn_output_weights = F.softmax(attn_output_weights, dim=-1)
|
||||
attn_output_weights = F.dropout(attn_output_weights,
|
||||
p=self.dropout,
|
||||
training=self.training)
|
||||
attn_output = torch.bmm(attn_output_weights, v)
|
||||
|
||||
return attn_output
|
||||
|
||||
|
||||
class DotProductAttentionStream(DotProductAttention):
|
||||
|
||||
def __init__(self, dropout=0.0, alpha=1.0):
|
||||
super(DotProductAttentionStream, self).__init__(dropout, alpha)
|
||||
|
||||
############################
|
||||
# Cache for stream inference
|
||||
############################
|
||||
self.k_weights_cache = None
|
||||
self.k_pos_weights_cache = None
|
||||
self.QK_exp_cache = None
|
||||
self.recache_cnt = 0
|
||||
self.recache_freq = 99999
|
||||
self.alpha = alpha
|
||||
|
||||
def clear_cache(self):
|
||||
self.k_weights_cache = None
|
||||
self.k_pos_weights_cache = None
|
||||
self.QK_exp_cache = None
|
||||
self.recache_cnt = 0
|
||||
|
||||
|
||||
def stream_inference(self, q, k, v, k_pos, v_pos, attn_mask=None, cache_num=1, cache_id=0):
|
||||
'''
|
||||
if self.k_weights_cache is not None:
|
||||
k_weights_new = torch.bmm(q, k[:, [-1]].transpose(1, 2))
|
||||
k_weights = torch.cat((self.k_weights_cache[:, :, 1:], k_weights_new), dim=-1)
|
||||
self.k_weights_cache = k_weights
|
||||
# k_pos_weights = self.k_pos_weights_cache
|
||||
else:
|
||||
k_weights = torch.bmm(q, k.transpose(1, 2))
|
||||
self.k_weights_cache = k_weights
|
||||
# k_pos_weights = torch.bmm(q, k_pos.transpose(1, 2))
|
||||
# self.k_pos_weights_cache = k_pos_weights
|
||||
# attn_output_weights = k_weights + k_pos_weights
|
||||
attn_output_weights = k_weights
|
||||
|
||||
if attn_mask is not None:
|
||||
attn_output_weights += attn_mask
|
||||
|
||||
attn_output_weights = F.softmax(attn_output_weights, dim=-1)
|
||||
attn_output_weights = F.dropout(attn_output_weights,
|
||||
p=self.dropout,
|
||||
training=self.training)
|
||||
# attn_output = torch.bmm(attn_output_weights, (v + v_pos))
|
||||
attn_output = torch.bmm(attn_output_weights, v)
|
||||
return attn_output
|
||||
'''
|
||||
B = 1
|
||||
num = 1
|
||||
q = q.repeat(B, 1, 1)
|
||||
k = k.repeat(B, num, 1)
|
||||
v = v.repeat(B, num, 1)
|
||||
attn_mask = attn_mask.repeat(B, 1, num)
|
||||
if self.alpha <= 1.0:
|
||||
if self.QK_exp_cache is None:
|
||||
QK = torch.bmm(q, k.transpose(1, 2)) # q~(H, Tq, C), k~(H, Tk, C) => O(H*Tq*Tk*C)
|
||||
self.QK_max = QK.max(dim=-1, keepdim=True).values
|
||||
QK_exp = torch.exp(QK - self.QK_max + attn_mask) # => O(H*Tq*Tk)
|
||||
self.QK_exp_cache = [QK_exp for _ in range(cache_num)]
|
||||
self.QKV_exp_cache = [torch.bmm(QK_exp, v) for _ in range(cache_num)] # => O(H*Tq*Tk*C)
|
||||
self.Z = [QK_exp.sum(dim=-1, keepdim=True) for _ in range(cache_num)] # => O(H*Tq*Tk)
|
||||
output = self.QKV_exp_cache[cache_id] / self.Z[cache_id]
|
||||
self.recache_cnt += 1
|
||||
return output
|
||||
else:
|
||||
QK_push = torch.bmm(q, k[:, -num:, :].transpose(1, 2)) # => O(H*Tq*C)
|
||||
QK_exp_push = torch.exp(QK_push - self.QK_max) # => O(H*Tq*Tk)
|
||||
sum_push = QK_exp_push.sum(-1, keepdim=True)
|
||||
QKV_exp_push = torch.bmm(QK_exp_push, v[:, -num:, :]) # => O(H*Tq*Tk)
|
||||
self.QKV_exp_cache[cache_id] = self.alpha * self.QKV_exp_cache[cache_id] + QKV_exp_push # => O(H*Tq*Tk)
|
||||
self.Z[cache_id] = self.alpha * self.Z[cache_id] + sum_push
|
||||
|
||||
output = self.QKV_exp_cache[cache_id] / self.Z[cache_id]
|
||||
self.recache_cnt += 1
|
||||
return output
|
||||
|
||||
if self.QK_exp_cache is None:
|
||||
QK = torch.bmm(q, k.transpose(1, 2)) # q~(H, Tq, C), k~(H, Tk, C) => O(H*Tq*Tk*C)
|
||||
self.QK_max = QK.max(dim=-1, keepdim=True).values
|
||||
QK_exp = torch.exp(QK - self.QK_max + attn_mask) # => O(H*Tq*Tk)
|
||||
self.QK_exp_cache = [QK_exp for _ in range(cache_num)]
|
||||
self.QKV_exp_cache = [torch.bmm(QK_exp, v) for _ in range(cache_num)] # => O(H*Tq*Tk*C)
|
||||
|
||||
self.Z = [QK_exp.sum(dim=-1, keepdim=True) for _ in range(cache_num)] # => O(H*Tq*Tk)
|
||||
self.QKV_exp_pop = [torch.bmm(self.QK_exp_cache[cache_i][:, :, :num], v[:, :num, :]) for cache_i in range(cache_num)]
|
||||
self.sum_pop = [self.QK_exp_cache[cache_i][:, :, :num].sum(-1, keepdim=True) for cache_i in range(cache_num)]
|
||||
|
||||
output = self.QKV_exp_cache[cache_id] / self.Z[cache_id]
|
||||
self.recache_cnt += 1
|
||||
return output
|
||||
elif (self.recache_cnt // cache_num) % self.recache_freq == 0:
|
||||
QK = torch.bmm(q, k.transpose(1, 2)) # q~(H, Tq, C), k~(H, Tk, C) => O(H*Tq*Tk*C)
|
||||
self.QK_max = QK.max(dim=-1, keepdim=True).values
|
||||
QK_exp = torch.exp(QK - self.QK_max + attn_mask) # => O(H*Tq*Tk)
|
||||
self.QK_exp_cache[cache_id] = QK_exp
|
||||
self.QKV_exp_cache[cache_id] = torch.bmm(QK_exp, v) # => O(H*Tq*Tk*C)
|
||||
|
||||
self.Z[cache_id] = QK_exp.sum(dim=-1, keepdim=True) # => O(H*Tq*Tk)
|
||||
|
||||
self.QKV_exp_pop[cache_id] = torch.bmm(self.QK_exp_cache[cache_id][:, :, :num], v[:, :num, :])
|
||||
self.sum_pop[cache_id] = self.QK_exp_cache[cache_id][:, :, :num].sum(-1, keepdim=True)
|
||||
output = self.QKV_exp_cache[cache_id] / self.Z[cache_id]
|
||||
self.recache_cnt += 1
|
||||
return output
|
||||
else:
|
||||
QK_push = torch.bmm(q, k[:, -num:, :].transpose(1, 2)) # => O(H*Tq*C)
|
||||
QK_exp_push = torch.exp(QK_push - self.QK_max) # => O(H*Tq*Tk)
|
||||
sum_push = QK_exp_push.sum(-1, keepdim=True)
|
||||
QKV_exp_push = torch.bmm(QK_exp_push, v[:, -num:, :]) # => O(H*Tq*Tk)
|
||||
|
||||
# add with no decay
|
||||
self.QKV_exp_cache[cache_id] = self.QKV_exp_cache[cache_id] + QKV_exp_push # => O(H*Tq*Tk)
|
||||
|
||||
# add with no decay
|
||||
self.Z[cache_id] = self.Z[cache_id] + sum_push # => O(H*Tq*Tk)
|
||||
|
||||
self.QK_exp_cache[cache_id] = torch.cat((self.QK_exp_cache[cache_id][:, :, num:], QK_exp_push), dim=-1)
|
||||
self.QKV_exp_pop[cache_id] = torch.bmm(self.QK_exp_cache[cache_id][:, :, :num], v[:, :num, :])
|
||||
self.sum_pop[cache_id] = self.QK_exp_cache[cache_id][:, :, :num].sum(-1, keepdim=True)
|
||||
output = self.QKV_exp_cache[cache_id] / self.Z[cache_id]
|
||||
self.recache_cnt += 1
|
||||
return output
|
||||
|
||||
|
||||
class MultiheadAttention(nn.Module):
|
||||
|
||||
def __init__(self, embed_dim, num_heads, dropout=0.0, bias=True, kdim=None, vdim=None,
|
||||
attention_type='dotproduct', decay_alpha=1.0):
|
||||
super(MultiheadAttention, self).__init__()
|
||||
|
||||
self.embed_dim = embed_dim
|
||||
self.num_heads = num_heads
|
||||
self.kdim = kdim if kdim is not None else embed_dim
|
||||
self.vdim = vdim if vdim is not None else embed_dim
|
||||
self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim
|
||||
|
||||
if self._qkv_same_embed_dim:
|
||||
self.in_proj_weight = nn.Parameter(torch.empty(3 * embed_dim, embed_dim))
|
||||
else:
|
||||
raise RuntimeError('Do not support q, k, v have different dimensions')
|
||||
|
||||
if bias:
|
||||
self.in_proj_bias = nn.Parameter(torch.empty(3 * embed_dim))
|
||||
else:
|
||||
self.register_parameter('in_proj_bias', None)
|
||||
|
||||
self.out_proj = nn.Linear(embed_dim, embed_dim)
|
||||
|
||||
if self._qkv_same_embed_dim:
|
||||
nn.init.xavier_uniform_(self.in_proj_weight)
|
||||
|
||||
if self.in_proj_bias is not None:
|
||||
nn.init.constant_(self.in_proj_bias, 0.)
|
||||
nn.init.constant_(self.out_proj.bias, 0.)
|
||||
|
||||
if attention_type == 'dotproduct':
|
||||
self.attention = DotProductAttention(dropout, decay_alpha, num_heads)
|
||||
elif attention_type == 'linear':
|
||||
self.attention = LinearAttention(dropout)
|
||||
else:
|
||||
raise RuntimeError('attention_type should be [dotproduct | linear]')
|
||||
|
||||
def forward(self, q, k, v, attn_mask=None, key_padding_mask=None):
|
||||
tsz, bsz, embed_dim = q.shape[0], q.shape[1], q.shape[2]
|
||||
|
||||
head_dim = embed_dim // self.num_heads
|
||||
assert head_dim * self.num_heads == embed_dim, \
|
||||
'embed_dim must be divisible by num_heads'
|
||||
scaling = float(head_dim) ** -0.5
|
||||
|
||||
_b = self.in_proj_bias
|
||||
_start = None
|
||||
_end = embed_dim
|
||||
_w = self.in_proj_weight[:_end, :]
|
||||
if _b is not None:
|
||||
_b = _b[:_end]
|
||||
q = F.linear(q, _w, _b)
|
||||
|
||||
_b = self.in_proj_bias
|
||||
_start = embed_dim
|
||||
_end = embed_dim * 2
|
||||
_w = self.in_proj_weight[_start:_end, :]
|
||||
if _b is not None:
|
||||
_b = _b[_start:_end]
|
||||
k = F.linear(k, _w, _b)
|
||||
|
||||
_b = self.in_proj_bias
|
||||
_start = embed_dim * 2
|
||||
_end = None
|
||||
_w = self.in_proj_weight[_start:, :]
|
||||
if _b is not None:
|
||||
_b = _b[_start:]
|
||||
v = F.linear(v, _w, _b)
|
||||
|
||||
q = q * scaling
|
||||
|
||||
q = q.contiguous().view(-1, bsz * self.num_heads, head_dim).transpose(0, 1)
|
||||
k = k.contiguous().view(-1, bsz * self.num_heads, head_dim).transpose(0, 1)
|
||||
v = v.contiguous().view(-1, bsz * self.num_heads, head_dim).transpose(0, 1)
|
||||
|
||||
if attn_mask is not None:
|
||||
attn_mask = attn_mask.unsqueeze(0).repeat(bsz, 1, 1)
|
||||
attn_mask = attn_mask.unsqueeze(1).repeat(1, self.num_heads, 1, 1)
|
||||
attn_mask = attn_mask.reshape(-1, *attn_mask.shape[2:])
|
||||
|
||||
if key_padding_mask is not None:
|
||||
if key_padding_mask.ndim == 4:
|
||||
key_padding_mask = key_padding_mask.unsqueeze(1).repeat(1, tsz, 1, 1, 1)
|
||||
key_padding_mask = key_padding_mask.unsqueeze(1).repeat(1, self.num_heads, 1, 1, 1, 1)
|
||||
key_padding_mask = key_padding_mask.reshape(*key_padding_mask.shape[:3], -1)
|
||||
else:
|
||||
key_padding_mask = key_padding_mask.unsqueeze(1).repeat(1, tsz, 1)
|
||||
key_padding_mask = key_padding_mask.unsqueeze(1).repeat(1, self.num_heads, 1, 1)
|
||||
key_padding_mask = key_padding_mask.reshape(-1, *key_padding_mask.shape[2:])
|
||||
|
||||
if attn_mask is not None and key_padding_mask is not None:
|
||||
mask = attn_mask + key_padding_mask
|
||||
elif attn_mask is not None:
|
||||
mask = attn_mask
|
||||
elif key_padding_mask is not None:
|
||||
mask = key_padding_mask
|
||||
else:
|
||||
mask = None
|
||||
|
||||
attn_output = self.attention(q, k, v, mask)
|
||||
attn_output = attn_output.transpose(0, 1).contiguous().view(tsz, bsz,
|
||||
self.embed_dim)
|
||||
return self.out_proj(attn_output), None
|
||||
|
||||
|
||||
class MultiheadAttentionStream(MultiheadAttention):
|
||||
|
||||
def __init__(self, embed_dim, num_heads, dropout=0.0, bias=True, kdim=None, vdim=None,
|
||||
attention_type='dotproduct', decay_alpha=1.0):
|
||||
super(MultiheadAttentionStream, self).__init__(embed_dim, num_heads,
|
||||
dropout, bias, kdim,
|
||||
vdim, attention_type,
|
||||
decay_alpha)
|
||||
|
||||
if attention_type == 'dotproduct':
|
||||
self.attention = DotProductAttentionStream(dropout, decay_alpha)
|
||||
elif attention_type == 'linear':
|
||||
self.attention = LinearAttentionStream(dropout)
|
||||
else:
|
||||
raise RuntimeError('attention_type should be [dotproduct | linear]')
|
||||
|
||||
############################
|
||||
# Cache for stream inference
|
||||
############################
|
||||
self.q_cache = None
|
||||
self.k_cache = None
|
||||
self.v_cache = None
|
||||
self.k_pos_cache = None
|
||||
self.v_pos_cache = None
|
||||
|
||||
def clear_cache(self):
|
||||
self.q_cache = None
|
||||
self.k_cache = None
|
||||
self.v_cache = None
|
||||
self.k_pos_cache = None
|
||||
self.v_pos_cache = None
|
||||
self.attention.clear_cache()
|
||||
|
||||
def stream_inference(self, q, k, v, pos, attn_mask=None, key_padding_mask=None,
|
||||
cache_num=1, cache_id=0):
|
||||
tsz, bsz, embed_dim = q.shape[0], q.shape[1], q.shape[2]
|
||||
|
||||
head_dim = embed_dim // self.num_heads
|
||||
assert head_dim * self.num_heads == embed_dim, \
|
||||
'embed_dim must be divisible by num_heads'
|
||||
scaling = float(head_dim) ** -0.5
|
||||
|
||||
if self.q_cache is not None:
|
||||
q = self.q_cache
|
||||
else:
|
||||
_b = self.in_proj_bias
|
||||
_start = None
|
||||
_end = embed_dim
|
||||
_w = self.in_proj_weight[:_end, :]
|
||||
if _b is not None:
|
||||
_b = _b[:_end]
|
||||
q = F.linear(q, _w, _b)
|
||||
self.q_cache = q
|
||||
|
||||
assert (self.k_cache is None) == (self.k_pos_cache is None)
|
||||
if self.k_cache is not None:
|
||||
_b = self.in_proj_bias
|
||||
_start = embed_dim
|
||||
_end = embed_dim * 2
|
||||
_w = self.in_proj_weight[_start:_end, :]
|
||||
if _b is not None:
|
||||
_b = _b[_start:_end]
|
||||
k_new = F.linear(k[[-1]], _w, None)
|
||||
k = torch.cat((self.k_cache[1:], k_new))
|
||||
self.k_cache = k
|
||||
k_pos = self.k_pos_cache
|
||||
else:
|
||||
_b = self.in_proj_bias
|
||||
_start = embed_dim
|
||||
_end = embed_dim * 2
|
||||
_w = self.in_proj_weight[_start:_end, :]
|
||||
if _b is not None:
|
||||
_b = _b[_start:_end]
|
||||
k = F.linear(k, _w, None)
|
||||
self.k_cache = k
|
||||
k_pos = F.linear(pos, _w, _b)
|
||||
self.k_pos_cache = k_pos
|
||||
|
||||
assert (self.v_cache is None) == (self.v_pos_cache is None)
|
||||
if self.v_cache is not None:
|
||||
_b = self.in_proj_bias
|
||||
_start = embed_dim * 2
|
||||
_end = None
|
||||
_w = self.in_proj_weight[_start:, :]
|
||||
if _b is not None:
|
||||
_b = _b[_start:]
|
||||
v_new = F.linear(v[[-1]], _w, None)
|
||||
v = torch.cat((self.v_cache[1:], v_new))
|
||||
self.v_cache = v
|
||||
v_pos = self.v_pos_cache
|
||||
else:
|
||||
_b = self.in_proj_bias
|
||||
_start = embed_dim * 2
|
||||
_end = None
|
||||
_w = self.in_proj_weight[_start:, :]
|
||||
if _b is not None:
|
||||
_b = _b[_start:]
|
||||
v = F.linear(v, _w, None)
|
||||
self.v_cache = v
|
||||
v_pos = F.linear(pos, _w, _b)
|
||||
self.v_pos_cache = v_pos
|
||||
|
||||
q = q * scaling
|
||||
|
||||
q = q.contiguous().view(-1, bsz * self.num_heads, head_dim).transpose(0, 1)
|
||||
k = k.contiguous().view(-1, bsz * self.num_heads, head_dim).transpose(0, 1)
|
||||
v = v.contiguous().view(-1, bsz * self.num_heads, head_dim).transpose(0, 1)
|
||||
k_pos = k_pos.contiguous().view(-1, bsz * self.num_heads, head_dim).transpose(0, 1)
|
||||
v_pos = v_pos.contiguous().view(-1, bsz * self.num_heads, head_dim).transpose(0, 1)
|
||||
|
||||
if attn_mask is not None:
|
||||
attn_mask = attn_mask.unsqueeze(0).repeat(bsz, 1, 1)
|
||||
attn_mask = attn_mask.unsqueeze(1).repeat(1, self.num_heads, 1, 1)
|
||||
attn_mask = attn_mask.reshape(-1, *attn_mask.shape[2:])
|
||||
|
||||
if key_padding_mask is not None:
|
||||
key_padding_mask = key_padding_mask.unsqueeze(1).repeat(1, tsz, 1)
|
||||
key_padding_mask = key_padding_mask.unsqueeze(1).repeat(1, self.num_heads, 1, 1)
|
||||
key_padding_mask = key_padding_mask.reshape(-1, *key_padding_mask.shape[2:])
|
||||
|
||||
if attn_mask is not None and key_padding_mask is not None:
|
||||
mask = attn_mask + key_padding_mask
|
||||
elif attn_mask is not None:
|
||||
mask = attn_mask
|
||||
elif key_padding_mask is not None:
|
||||
mask = key_padding_mask
|
||||
else:
|
||||
mask = None
|
||||
|
||||
attn_output = self.attention.stream_inference(q, k, v, k_pos, v_pos, mask,
|
||||
cache_num=cache_num, cache_id=cache_id)
|
||||
attn_output = attn_output.transpose(0, 1).contiguous().view(tsz, bsz,
|
||||
self.embed_dim)
|
||||
return self.out_proj(attn_output), None
|
||||
|
||||
|
||||
class LinearAttention(nn.Module):
|
||||
|
||||
def __init__(self, dropout=0.0, nonlinear=F.elu,
|
||||
eps=1e-6):
|
||||
super(LinearAttention, self).__init__()
|
||||
|
||||
self.dropout = dropout
|
||||
self.nonlinear = nonlinear
|
||||
self.eps = eps
|
||||
|
||||
def forward(self, q, k, v, attn_mask=None):
|
||||
kk = self.nonlinear(k) + 1
|
||||
qq = self.nonlinear(q) + 1
|
||||
vv = self.nonlinear(v) + 1
|
||||
|
||||
kv = torch.bmm(kk.transpose(1, 2), vv)
|
||||
|
||||
ks = torch.sum(kk, dim=1, keepdim=True) # (B 1 C)
|
||||
Z = 1 / (torch.bmm(qq, ks.transpose(1, 2)) + self.eps)
|
||||
attn_output = torch.bmm(qq, kv.transpose(1, 2)) * Z
|
||||
|
||||
# if attn_mask is not None:
|
||||
# print(attn_mask.shape)
|
||||
# raise RuntimeError(("LinearAttention does not support arbitrary "
|
||||
# "attention masks"))
|
||||
|
||||
return attn_output
|
||||
|
||||
class LinearAttentionStream(LinearAttention):
|
||||
|
||||
def __init__(self, dropout=0.0):
|
||||
super(LinearAttentionStream, self).__init__(dropout)
|
||||
|
||||
############################
|
||||
# Cache for stream inference
|
||||
############################
|
||||
self.k_weights_cache = None
|
||||
self.k_pos_weights_cache = None
|
||||
|
||||
def stream_inference(self, q, k, v, k_pos, v_pos, attn_mask=None):
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
class PositionalEncoding(nn.Module):
|
||||
|
||||
def __init__(self, d_model, dropout=0.1, max_len=5000):
|
||||
super(PositionalEncoding, self).__init__()
|
||||
|
||||
self.dropout = nn.Dropout(p=dropout)
|
||||
|
||||
pe = torch.zeros(max_len, d_model)
|
||||
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
|
||||
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
|
||||
pe[:, 0::2] = torch.sin(position * div_term)
|
||||
pe[:, 1::2] = torch.cos(position * div_term)
|
||||
pe = pe.unsqueeze(0).transpose(0, 1)
|
||||
self.register_buffer('pe', pe)
|
||||
|
||||
def forward(self, x, padding=0):
|
||||
x = x + self.pe[padding: padding + x.shape[0], :]
|
||||
return self.dropout(x)
|
||||
|
||||
|
||||
class PositionalEncoding2D(nn.Module):
|
||||
|
||||
def __init__(self, d_model, dropout=0.1, height=64, width=64):
|
||||
super(PositionalEncoding2D, self).__init__()
|
||||
|
||||
self.dropout = nn.Dropout(p=dropout)
|
||||
|
||||
pe = torch.zeros(height, width, d_model)
|
||||
position_x = torch.arange(0, width, dtype=torch.float).unsqueeze(1)
|
||||
position_y = torch.arange(0, height, dtype=torch.float).unsqueeze(1)
|
||||
d_model = int(d_model / 2)
|
||||
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
|
||||
pe[:, :, 0:d_model:2] = torch.sin(position_x * div_term).unsqueeze(0).repeat(height, 1, 1)
|
||||
pe[:, :, 1:d_model:2] = torch.cos(position_x * div_term).unsqueeze(0).repeat(height, 1, 1)
|
||||
pe[:, :, d_model::2] = torch.sin(position_y * div_term).unsqueeze(1).repeat(1, width, 1)
|
||||
pe[:, :, d_model + 1::2] = torch.cos(position_y * div_term).unsqueeze(1).repeat(1, width, 1)
|
||||
pe = pe.unsqueeze(0).unsqueeze(0) #.transpose(0, 1)
|
||||
self.register_buffer('pe', pe)
|
||||
|
||||
def forward(self, x, padding=(0,0)):
|
||||
x = x + self.pe[:, :,
|
||||
padding[0]: padding[0] + x.shape[2],
|
||||
padding[1]: padding[1] + x.shape[3], :]
|
||||
return self.dropout(x)
|
||||
250
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/transformer/transformer.py
vendored
Normal file
250
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/transformer/transformer.py
vendored
Normal file
@@ -0,0 +1,250 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import copy
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from .multihead_attention import MultiheadAttentionStream as MultiheadAttention
|
||||
|
||||
|
||||
class Transformer(nn.Module):
|
||||
|
||||
def __init__(self, d_model=512, nhead=8, num_encoder_layers=6,
|
||||
num_decoder_layers=6, dim_feedforward=2048, dropout=0.1,
|
||||
activation='relu', custom_encoder=None, custom_decoder=None):
|
||||
super(Transformer, self).__init__()
|
||||
|
||||
if custom_encoder is not None:
|
||||
self.encoder = custom_encoder
|
||||
else:
|
||||
encoder_layer = TransformerEncoderLayer(d_model, nhead, dim_feedforward, dropout, activation)
|
||||
encoder_norm = nn.LayerNorm(d_model)
|
||||
self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm)
|
||||
|
||||
if custom_decoder is not None:
|
||||
self.decoder = custom_decoder
|
||||
else:
|
||||
decoder_layer = TransformerDecoderLayer(d_model, nhead, dim_feedforward, dropout, activation)
|
||||
decoder_norm = nn.LayerNorm(d_model)
|
||||
self.decoder = TransformerDecoder(decoder_layer, num_decoder_layers, decoder_norm)
|
||||
|
||||
self.d_model = d_model
|
||||
self.nhead = nhead
|
||||
|
||||
def forward(self, src, tgt, src_mask=None, tgt_mask=None,
|
||||
memory_mask=None, src_key_padding_mask=None,
|
||||
tgt_key_padding_mask=None, memory_key_padding_mask=None):
|
||||
if src.size(1) != tgt.size(1):
|
||||
raise RuntimeError('the batch number of src and tgt must be equal')
|
||||
|
||||
if src.size(2) != self.d_model or tgt.size(2) != self.d_model:
|
||||
raise RuntimeError('the feature number of src and tgt must be equal to d_model')
|
||||
|
||||
memory = self.encoder(src, mask=src_mask, src_key_padding_mask=src_key_padding_mask)
|
||||
output = self.decoder(tgt, memory, tgt_mask=tgt_mask, memory_mask=memory_mask,
|
||||
tgt_key_padding_mask=tgt_key_padding_mask,
|
||||
memory_key_padding_mask=memory_key_padding_mask)
|
||||
return output
|
||||
|
||||
|
||||
class TransformerEncoder(nn.Module):
|
||||
|
||||
def __init__(self, encoder_layer, num_layers, norm=None):
|
||||
super(TransformerEncoder, self).__init__()
|
||||
|
||||
self.layers = _get_clones(encoder_layer, num_layers)
|
||||
self.num_layers = num_layers
|
||||
self.norm = norm
|
||||
|
||||
def forward(self, src, src_mask=None, src_key_padding_mask=None):
|
||||
output = src
|
||||
|
||||
for mod in self.layers:
|
||||
output = mod(output, src_mask=src_mask,
|
||||
src_key_padding_mask=src_key_padding_mask)
|
||||
|
||||
if self.norm is not None:
|
||||
output = self.norm(output)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
class TransformerDecoder(nn.Module):
|
||||
|
||||
def __init__(self, decoder_layer, num_layers, norm=None):
|
||||
super(TransformerDecoder, self).__init__()
|
||||
|
||||
self.layers = _get_clones(decoder_layer, num_layers)
|
||||
self.num_layers = num_layers
|
||||
self.norm = norm
|
||||
|
||||
def clear_cache(self):
|
||||
if len(self.layers) != 1:
|
||||
raise RuntimeError('Number of layers cannot larger than 1 for stream inference')
|
||||
|
||||
self.layers[0].clear_cache()
|
||||
|
||||
def stream_inference(self, tgt, memory, pos, tgt_mask=None,
|
||||
memory_mask=None, tgt_key_padding_mask=None,
|
||||
memory_key_padding_mask=None, cache_num=1, cache_id=0):
|
||||
output = tgt
|
||||
|
||||
if len(self.layers) != 1:
|
||||
raise RuntimeError('Number of layers cannot larger than 1 for stream inference')
|
||||
|
||||
output = self.layers[0].stream_inference(output, memory, pos,
|
||||
tgt_mask=tgt_mask,
|
||||
memory_mask=memory_mask,
|
||||
tgt_key_padding_mask=tgt_key_padding_mask,
|
||||
memory_key_padding_mask=memory_key_padding_mask,
|
||||
cache_num=cache_num, cache_id=cache_id)
|
||||
|
||||
if self.norm is not None:
|
||||
output = self.norm(output)
|
||||
|
||||
return output
|
||||
|
||||
def forward(self, tgt, memory, tgt_mask=None,
|
||||
memory_mask=None, tgt_key_padding_mask=None,
|
||||
memory_key_padding_mask=None):
|
||||
output = tgt
|
||||
|
||||
for mod in self.layers:
|
||||
output = mod(output, memory, tgt_mask=tgt_mask,
|
||||
memory_mask=memory_mask,
|
||||
tgt_key_padding_mask=tgt_key_padding_mask,
|
||||
memory_key_padding_mask=memory_key_padding_mask)
|
||||
|
||||
if self.norm is not None:
|
||||
output = self.norm(output)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
class TransformerEncoderLayer(nn.Module):
|
||||
|
||||
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation='relu'):
|
||||
super(TransformerEncoderLayer, self).__init__()
|
||||
|
||||
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout)
|
||||
|
||||
# Implementation of Feedforward model
|
||||
self.linear1 = nn.Linear(d_model, dim_feedforward)
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.linear2 = nn.Linear(dim_feedforward, d_model)
|
||||
|
||||
self.norm1 = nn.LayerNorm(d_model)
|
||||
self.norm2 = nn.LayerNorm(d_model)
|
||||
self.dropout1 = nn.Dropout(dropout)
|
||||
self.dropout2 = nn.Dropout(dropout)
|
||||
|
||||
self.activation = _get_activation_fn(activation)
|
||||
|
||||
def __setstate__(self, state):
|
||||
if 'activation' not in state:
|
||||
state['activation'] = F.relu
|
||||
super(TransformerEncoderLayer, self).__setstate__(state)
|
||||
|
||||
def forward(self, src, src_mask=None, src_key_padding_mask=None):
|
||||
src2 = self.self_attn(src, src, src, attn_mask=src_mask,
|
||||
key_padding_mask=src_key_padding_mask)[0]
|
||||
src = src + self.dropout1(src2)
|
||||
src = self.norm1(src)
|
||||
src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
|
||||
src = src + self.dropout2(src2)
|
||||
src = self.norm2(src)
|
||||
return src
|
||||
|
||||
|
||||
class TransformerDecoderLayer(nn.Module):
|
||||
|
||||
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1,
|
||||
activation='relu', attention_type='dotproduct',
|
||||
decay_alpha=1.0):
|
||||
super(TransformerDecoderLayer, self).__init__()
|
||||
|
||||
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout,
|
||||
attention_type=attention_type)
|
||||
self.multihead_attn = MultiheadAttention(d_model, nhead, dropout=dropout,
|
||||
attention_type=attention_type,
|
||||
decay_alpha=decay_alpha)
|
||||
|
||||
# Implementation of Feedforward model
|
||||
self.linear1 = nn.Linear(d_model, dim_feedforward)
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.linear2 = nn.Linear(dim_feedforward, d_model)
|
||||
|
||||
self.norm1 = nn.LayerNorm(d_model)
|
||||
self.norm2 = nn.LayerNorm(d_model)
|
||||
self.norm3 = nn.LayerNorm(d_model)
|
||||
self.dropout1 = nn.Dropout(dropout)
|
||||
self.dropout2 = nn.Dropout(dropout)
|
||||
self.dropout3 = nn.Dropout(dropout)
|
||||
|
||||
self.activation = _get_activation_fn(activation)
|
||||
|
||||
############################
|
||||
# Cache for stream inference
|
||||
############################
|
||||
self.tgt_cache = None
|
||||
|
||||
def __setstate__(self, state):
|
||||
if 'activation' not in state:
|
||||
state['activation'] = F.relu
|
||||
super(TransformerDecoderLayer, self).__setstate__(state)
|
||||
|
||||
def clear_cache(self):
|
||||
self.tgt_cache = None
|
||||
self.multihead_attn.clear_cache()
|
||||
|
||||
def stream_inference(self, tgt, memory, pos, tgt_mask=None, memory_mask=None,
|
||||
tgt_key_padding_mask=None, memory_key_padding_mask=None,
|
||||
cache_num=1, cache_id=0):
|
||||
if self.tgt_cache is None:
|
||||
tgt2 = self.self_attn(tgt, tgt, tgt, attn_mask=tgt_mask,
|
||||
key_padding_mask=tgt_key_padding_mask)[0]
|
||||
tgt = tgt + self.dropout1(tgt2)
|
||||
tgt = self.norm1(tgt)
|
||||
self.tgt_cache = tgt
|
||||
else:
|
||||
tgt = self.tgt_cache
|
||||
tgt2 = self.multihead_attn.stream_inference(tgt, memory, memory, pos, attn_mask=memory_mask,
|
||||
key_padding_mask=memory_key_padding_mask,
|
||||
cache_num=cache_num, cache_id=cache_id)[0]
|
||||
tgt = tgt + self.dropout2(tgt2)
|
||||
tgt = self.norm2(tgt)
|
||||
tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
|
||||
tgt = tgt + self.dropout3(tgt2)
|
||||
tgt = self.norm3(tgt)
|
||||
return tgt
|
||||
|
||||
def forward(self, tgt, memory, tgt_mask=None, memory_mask=None,
|
||||
tgt_key_padding_mask=None, memory_key_padding_mask=None):
|
||||
tgt2 = self.self_attn(tgt, tgt, tgt, attn_mask=tgt_mask,
|
||||
key_padding_mask=tgt_key_padding_mask)[0]
|
||||
tgt = tgt + self.dropout1(tgt2)
|
||||
tgt = self.norm1(tgt)
|
||||
tgt2 = self.multihead_attn(tgt, memory, memory, attn_mask=memory_mask,
|
||||
key_padding_mask=memory_key_padding_mask)[0]
|
||||
tgt = tgt + self.dropout2(tgt2)
|
||||
tgt = self.norm2(tgt)
|
||||
tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
|
||||
tgt = tgt + self.dropout3(tgt2)
|
||||
tgt = self.norm3(tgt)
|
||||
return tgt
|
||||
|
||||
|
||||
def _get_clones(module, N):
|
||||
return nn.ModuleList([copy.deepcopy(module) for i in range(N)])
|
||||
|
||||
|
||||
def _get_activation_fn(activation):
|
||||
if activation == 'relu':
|
||||
return F.relu
|
||||
elif activation == 'gelu':
|
||||
return F.gelu
|
||||
|
||||
raise RuntimeError('activation should be relu/gelu, not {}'.format(activation))
|
||||
16
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/transformer/utils.py
vendored
Normal file
16
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/transformer/utils.py
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import torch
|
||||
assert torch.__version__ >= '1.6.0'
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
def layer_norm(d_model, condition=True):
|
||||
return nn.LayerNorm(d_model) if condition else None
|
||||
|
||||
|
||||
def generate_square_subsequent_mask(sz):
|
||||
mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
|
||||
mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
|
||||
return mask
|
||||
33
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/weights_init.py
vendored
Normal file
33
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/models/weights_init.py
vendored
Normal file
@@ -0,0 +1,33 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import math
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
def weights_init(m):
|
||||
if isinstance(m, nn.Linear):
|
||||
nn.init.kaiming_uniform_(m.weight.data, a=math.sqrt(5))
|
||||
elif isinstance(m, nn.Conv1d):
|
||||
nn.init.normal_(m.weight.data)
|
||||
if m.bias is not None:
|
||||
nn.init.normal_(m.bias.data)
|
||||
elif isinstance(m, nn.Conv2d):
|
||||
nn.init.xavier_normal_(m.weight.data)
|
||||
if m.bias is not None:
|
||||
nn.init.normal_(m.bias.data)
|
||||
elif isinstance(m, nn.ConvTranspose1d):
|
||||
nn.init.normal_(m.weight.data)
|
||||
if m.bias is not None:
|
||||
nn.init.normal_(m.bias.data)
|
||||
elif isinstance(m, nn.ConvTranspose2d):
|
||||
nn.init.xavier_normal_(m.weight.data)
|
||||
if m.bias is not None:
|
||||
nn.init.normal_(m.bias.data)
|
||||
elif isinstance(m, nn.BatchNorm1d):
|
||||
nn.init.normal_(m.weight.data, mean=1, std=0.02)
|
||||
nn.init.constant_(m.bias.data, 0)
|
||||
elif isinstance(m, nn.BatchNorm2d):
|
||||
nn.init.normal_(m.weight.data, mean=1, std=0.02)
|
||||
nn.init.constant_(m.bias.data, 0)
|
||||
5
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/optimizers/__init__.py
vendored
Normal file
5
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/optimizers/__init__.py
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from .optimizers import build_optimizer
|
||||
from .lr_scheduler import build_scheduler
|
||||
224
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/optimizers/lr_scheduler.py
vendored
Normal file
224
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/optimizers/lr_scheduler.py
vendored
Normal file
@@ -0,0 +1,224 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import math
|
||||
from bisect import bisect_left
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.optim.lr_scheduler import _LRScheduler
|
||||
|
||||
|
||||
def _get_warmup_factor_at_iter(warmup_method,
|
||||
this_iter,
|
||||
warmup_iters,
|
||||
warmup_factor):
|
||||
if this_iter >= warmup_iters:
|
||||
return 1.0
|
||||
|
||||
if warmup_method == 'constant':
|
||||
return warmup_factor
|
||||
elif warmup_method == 'linear':
|
||||
alpha = this_iter / warmup_iters
|
||||
return warmup_factor * (1 - alpha) + alpha
|
||||
else:
|
||||
raise ValueError('Unknown warmup method: {}'.format(warmup_method))
|
||||
|
||||
|
||||
class MultiStepLR(_LRScheduler):
|
||||
|
||||
def __init__(self,
|
||||
optimizer,
|
||||
milestones,
|
||||
gamma=0.1,
|
||||
last_epoch=-1):
|
||||
if not list(milestones) == sorted(milestones):
|
||||
raise ValueError(
|
||||
'Milestones should be a list of increasing integers. Got {}'.format(milestones)
|
||||
)
|
||||
self.milestones = milestones
|
||||
self.gamma = gamma
|
||||
super().__init__(optimizer, last_epoch)
|
||||
|
||||
def get_lr(self):
|
||||
return [
|
||||
base_lr
|
||||
* self.gamma ** bisect_left(self.milestones, self.last_epoch)
|
||||
for base_lr in self.base_lrs
|
||||
]
|
||||
|
||||
def _compute_values(self):
|
||||
# The new interface
|
||||
return self.get_lr()
|
||||
|
||||
|
||||
class WarmupMultiStepLR(_LRScheduler):
|
||||
|
||||
def __init__(self,
|
||||
optimizer,
|
||||
milestones,
|
||||
gamma=0.1,
|
||||
warmup_factor=0.3,
|
||||
warmup_iters=500,
|
||||
warmup_method='linear',
|
||||
last_epoch=-1):
|
||||
if not list(milestones) == sorted(milestones):
|
||||
raise ValueError(
|
||||
'Milestones should be a list of increasing integers. Got {}'.format(milestones)
|
||||
)
|
||||
self.milestones = milestones
|
||||
self.gamma = gamma
|
||||
self.warmup_factor = warmup_factor
|
||||
self.warmup_iters = warmup_iters
|
||||
self.warmup_method = warmup_method
|
||||
super().__init__(optimizer, last_epoch)
|
||||
|
||||
def get_lr(self):
|
||||
warmup_factor = _get_warmup_factor_at_iter(
|
||||
self.warmup_method,
|
||||
self.last_epoch,
|
||||
self.warmup_iters,
|
||||
self.warmup_factor,
|
||||
)
|
||||
import pdb; pdb.set_trace()
|
||||
return [
|
||||
base_lr
|
||||
* warmup_factor
|
||||
* self.gamma ** bisect_left(self.milestones, self.last_epoch)
|
||||
for base_lr in self.base_lrs
|
||||
]
|
||||
|
||||
def _compute_values(self):
|
||||
# The new interface
|
||||
return self.get_lr()
|
||||
|
||||
|
||||
class CosineLR(_LRScheduler):
|
||||
|
||||
def __init__(self,
|
||||
optimizer,
|
||||
max_iters,
|
||||
last_epoch=-1):
|
||||
self.max_iters = max_iters
|
||||
super().__init__(optimizer, last_epoch)
|
||||
|
||||
def get_lr(self):
|
||||
return [
|
||||
base_lr
|
||||
* 0.5
|
||||
* (1.0 + math.cos(math.pi * self.last_epoch / self.max_iters))
|
||||
for base_lr in self.base_lrs
|
||||
]
|
||||
|
||||
def _compute_values(self):
|
||||
# The new interface
|
||||
return self.get_lr()
|
||||
|
||||
|
||||
class WarmupCosineLR(_LRScheduler):
|
||||
|
||||
def __init__(self,
|
||||
optimizer,
|
||||
max_iters,
|
||||
gamma=0.1,
|
||||
warmup_factor=0.3,
|
||||
warmup_iters=500,
|
||||
warmup_method='linear',
|
||||
last_epoch=-1):
|
||||
self.max_iters = max_iters
|
||||
self.warmup_factor = warmup_factor
|
||||
self.warmup_iters = warmup_iters
|
||||
self.warmup_method = warmup_method
|
||||
super().__init__(optimizer, last_epoch)
|
||||
|
||||
def get_lr(self):
|
||||
warmup_factor = _get_warmup_factor_at_iter(
|
||||
self.warmup_method,
|
||||
self.last_epoch,
|
||||
self.warmup_iters,
|
||||
self.warmup_factor,
|
||||
)
|
||||
return [
|
||||
base_lr
|
||||
* warmup_factor
|
||||
* 0.5
|
||||
* (1.0 + math.cos(math.pi * self.last_epoch / self.max_iters))
|
||||
for base_lr in self.base_lrs
|
||||
]
|
||||
|
||||
def _compute_values(self):
|
||||
# The new interface
|
||||
return self.get_lr()
|
||||
|
||||
class DummyScheduler:
|
||||
def __init__(self, optimizer):
|
||||
self.optimizer = optimizer
|
||||
self.base_lrs = [group['lr'] for group in optimizer.param_groups]
|
||||
|
||||
def get_last_lr(self):
|
||||
return self.base_lrs
|
||||
|
||||
def step(self):
|
||||
pass # No operation for stepping the scheduler
|
||||
|
||||
|
||||
def build_scheduler(cfg, optimizer, num_iters_per_epoch):
|
||||
"""Unlike the PyTorch version, our schedulers adjust the learning rate
|
||||
according to iteration rather than epoch.
|
||||
"""
|
||||
# Set last epoch (here 'epoch' is actually 'iters')
|
||||
last_epoch = (cfg.SOLVER.START_EPOCH - 1) * num_iters_per_epoch
|
||||
|
||||
if cfg.SOLVER.SCHEDULER.SCHEDULER_NAME == 'none' or cfg.SOLVER.SCHEDULER.SCHEDULER_NAME is None:
|
||||
scheduler = DummyScheduler(optimizer)
|
||||
|
||||
elif cfg.SOLVER.SCHEDULER.SCHEDULER_NAME == 'multistep':
|
||||
# Convert milestones epochs to iters
|
||||
milestones = [(i - 1) * num_iters_per_epoch for i in cfg.SOLVER.SCHEDULER.MILESTONES]
|
||||
|
||||
scheduler = MultiStepLR(
|
||||
optimizer,
|
||||
milestones=milestones,
|
||||
gamma=cfg.SOLVER.SCHEDULER.GAMMA,
|
||||
last_epoch=last_epoch)
|
||||
elif cfg.SOLVER.SCHEDULER.SCHEDULER_NAME == 'warmup_multistep':
|
||||
# Convert milestones epochs to iters
|
||||
milestones = [(i - 1) * num_iters_per_epoch for i in cfg.SOLVER.SCHEDULER.MILESTONES]
|
||||
|
||||
# Convert warmup epochs to iters
|
||||
cfg.SOLVER.SCHEDULER.WARMUP_ITERS = int(cfg.SOLVER.SCHEDULER.WARMUP_EPOCHS * num_iters_per_epoch)
|
||||
|
||||
scheduler = WarmupMultiStepLR(
|
||||
optimizer,
|
||||
milestones=milestones,
|
||||
gamma=cfg.SOLVER.SCHEDULER.GAMMA,
|
||||
warmup_factor=cfg.SOLVER.SCHEDULER.WARMUP_FACTOR,
|
||||
warmup_iters=cfg.SOLVER.SCHEDULER.WARMUP_ITERS,
|
||||
warmup_method=cfg.SOLVER.SCHEDULER.WARMUP_METHOD,
|
||||
last_epoch=last_epoch)
|
||||
elif cfg.SOLVER.SCHEDULER.SCHEDULER_NAME == 'cosine':
|
||||
# Get max number of iters
|
||||
max_iters = cfg.SOLVER.NUM_EPOCHS * num_iters_per_epoch
|
||||
|
||||
scheduler = CosineLR(
|
||||
optimizer,
|
||||
max_iters=max_iters,
|
||||
last_epoch=last_epoch)
|
||||
elif cfg.SOLVER.SCHEDULER.SCHEDULER_NAME == 'warmup_cosine':
|
||||
# Get max number of iters
|
||||
max_iters = cfg.SOLVER.NUM_EPOCHS * num_iters_per_epoch
|
||||
|
||||
# Convert warmup epochs to iters
|
||||
cfg.SOLVER.SCHEDULER.WARMUP_ITERS = int(cfg.SOLVER.SCHEDULER.WARMUP_EPOCHS * num_iters_per_epoch)
|
||||
|
||||
scheduler = WarmupCosineLR(
|
||||
optimizer,
|
||||
max_iters=max_iters,
|
||||
gamma=cfg.SOLVER.SCHEDULER.GAMMA,
|
||||
warmup_factor=cfg.SOLVER.SCHEDULER.WARMUP_FACTOR,
|
||||
warmup_iters=cfg.SOLVER.SCHEDULER.WARMUP_ITERS,
|
||||
warmup_method=cfg.SOLVER.SCHEDULER.WARMUP_METHOD,
|
||||
last_epoch=last_epoch)
|
||||
else:
|
||||
raise RuntimeError('Unknown lr scheduler: {}'.format(cfg.SOLVER.SCHEDULER.SCHEDULER_NAME))
|
||||
return scheduler
|
||||
27
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/optimizers/optimizers.py
vendored
Normal file
27
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/optimizers/optimizers.py
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
__all__ = ['build_optimizer']
|
||||
|
||||
import torch.optim as optim
|
||||
|
||||
|
||||
def build_optimizer(cfg, model):
|
||||
if cfg.SOLVER.OPTIMIZER == 'sgd':
|
||||
optimizer = optim.SGD(
|
||||
[{'params': model.parameters(), 'initial_lr': cfg.SOLVER.BASE_LR}],
|
||||
lr=cfg.SOLVER.BASE_LR, weight_decay=cfg.SOLVER.WEIGHT_DECAY, momentum=cfg.SOLVER.MOMENTUM,
|
||||
)
|
||||
elif cfg.SOLVER.OPTIMIZER == 'adam':
|
||||
optimizer = optim.Adam(
|
||||
[{'params': model.parameters(), 'initial_lr': cfg.SOLVER.BASE_LR}],
|
||||
lr=cfg.SOLVER.BASE_LR, weight_decay=cfg.SOLVER.WEIGHT_DECAY,
|
||||
)
|
||||
elif cfg.SOLVER.OPTIMIZER == 'adamw':
|
||||
optimizer = optim.AdamW(
|
||||
[{'params': model.parameters(), 'initial_lr': cfg.SOLVER.BASE_LR}],
|
||||
lr=cfg.SOLVER.BASE_LR, weight_decay=cfg.SOLVER.WEIGHT_DECAY,
|
||||
)
|
||||
else:
|
||||
raise RuntimeError('Unknown optimizer: {}'.format(cfg.SOLVER.OPTIMIZER))
|
||||
return optimizer
|
||||
2
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/__init__.py
vendored
Normal file
2
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/__init__.py
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
144
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/checkpointer.py
vendored
Normal file
144
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/checkpointer.py
vendored
Normal file
@@ -0,0 +1,144 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
__all__ = ['setup_checkpointer']
|
||||
|
||||
import os.path as osp
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class Checkpointer(object):
|
||||
|
||||
def __init__(self, cfg, phase):
|
||||
|
||||
# Load pretrained checkpoint
|
||||
|
||||
self.is_pretrained = False
|
||||
self.cfg = cfg
|
||||
self.checkpoint = self._load_checkpoint(cfg.MODEL.CHECKPOINT)
|
||||
self.except_classifier = cfg.MODEL.EXCEPT_CLASSIFIER if phase == 'train' else False
|
||||
|
||||
if self.checkpoint is not None and phase == 'train':
|
||||
# further training with the same dataset: resume
|
||||
model_ckpt_str = cfg['MODEL']['CHECKPOINT'].replace('_', '')
|
||||
if cfg['DATA']['DATA_NAME'].lower() in model_ckpt_str.lower():
|
||||
cfg.SOLVER.START_EPOCH += self.checkpoint.get('epoch', 0)
|
||||
# further training with the different dataset: finetuning on pretrained weight
|
||||
else:
|
||||
self.is_pretrained = True
|
||||
print(f'WARNING) Finetuning on pretrained weight: {cfg.MODEL.CHECKPOINT}')
|
||||
|
||||
elif self.checkpoint is None and phase != 'train':
|
||||
# no checkpoint in infernece mode: error
|
||||
raise RuntimeError('Cannot find checkpoint {}'.format(cfg.MODEL.CHECKPOINT))
|
||||
|
||||
self.output_dir = cfg.OUTPUT_DIR
|
||||
|
||||
def load(self, model, optimizer=None):
|
||||
if self.checkpoint is not None:
|
||||
# try load the ckpt first, then if there is a size mismatch, do other operation
|
||||
if not self.is_pretrained: # just resume
|
||||
# Attempt to load the state dictionary from the checkpoint
|
||||
model.load_state_dict(self.checkpoint['model_state_dict'])
|
||||
# Load the optimizer state if provided
|
||||
if optimizer is not None:
|
||||
if 'optimizer_state_dict' in self.checkpoint:
|
||||
optimizer.load_state_dict(self.checkpoint['optimizer_state_dict'])
|
||||
else:
|
||||
print('Optimizer state dictionary not found in checkpoint.')
|
||||
|
||||
else:
|
||||
# different model structure (pretrained)
|
||||
print(f'WARNING) Load checkpoint separately due to possiblity of different model structure, src/rekognition_online_action_detection/utils/checkpointer.py L51')
|
||||
model_dict = model.state_dict()
|
||||
unnecessary_keys = []
|
||||
unmatched_keys = []
|
||||
matched_ckpt = {}
|
||||
|
||||
# loaded_state_dict = self.checkpoint['model_state_dict']
|
||||
|
||||
# # Initialize a flag to track if all values match
|
||||
# all_values_match = True
|
||||
|
||||
# # Iterate through all keys and compare values
|
||||
# for key in model_dict:
|
||||
# import pdb; pdb.set_trace()
|
||||
# if not torch.allclose(model_dict[key], loaded_state_dict[key]):
|
||||
# print(f"Mismatch found in {key}")
|
||||
# print(f"Model value: {model_dict[key]}")
|
||||
# print(f"Loaded value: {loaded_state_dict[key]}")
|
||||
# print(f"Difference: {model_dict[key] - loaded_state_dict[key]}")
|
||||
# print("---")
|
||||
# all_values_match = False
|
||||
|
||||
# Filter out unnecessary keys
|
||||
for k, v in self.checkpoint['model_state_dict'].items():
|
||||
if k not in model_dict:
|
||||
print(f'Unnecessary key: {k} not found in model')
|
||||
unnecessary_keys.append(k)
|
||||
elif model_dict[k].shape != v.shape:
|
||||
print(f'Unnecessary key: {k} shape {v.shape} does not match {model_dict[k].shape}')
|
||||
unmatched_keys.append(k)
|
||||
elif self.except_classifier is True and 'classifier' in k:
|
||||
print(f'Unnecessary key: {k} is classifier')
|
||||
unnecessary_keys.append(k)
|
||||
else:
|
||||
matched_ckpt[k] = v
|
||||
|
||||
print(f'Unloaded keys: {unnecessary_keys}')
|
||||
print(f'Unmathced keys: {unmatched_keys}')
|
||||
# Update the existing state dict with the matched keys
|
||||
# import pdb; pdb.set_trace()
|
||||
model_dict.update(matched_ckpt)
|
||||
model.load_state_dict(model_dict)
|
||||
# for unmatched_key in unmatched_keys:
|
||||
# loaded_dict_shape = self.checkpoint['model_state_dict'][unmatched_key].shape
|
||||
# model_dict_shape = model_dict[unmatched_key].shape
|
||||
# if len(loaded_dict_shape) == 2:
|
||||
# if loaded_dict_shape[0] != model_dict_shape[0]:
|
||||
# model_dict[unmatched_key][:loaded_dict_shape[0], :] = self.checkpoint['model_state_dict'][unmatched_key]
|
||||
# print(f'Updated key: {unmatched_key}, model_dict[{unmatched_key}][:{loaded_dict_shape[0]}, :] = checkpoint["model_state_dict"][{unmatched_key}]')
|
||||
# else:
|
||||
# model_dict[unmatched_key][:, :loaded_dict_shape[1]] = self.checkpoint['model_state_dict'][unmatched_key]
|
||||
# print(f'Updated key: {unmatched_key}, model_dict[{unmatched_key}][:, :{loaded_dict_shape[1]}] = checkpoint["model_state_dict"][{unmatched_key}]')
|
||||
# elif len(loaded_dict_shape) == 1:
|
||||
# model_dict[unmatched_key][:loaded_dict_shape[0]] = self.checkpoint['model_state_dict'][unmatched_key]
|
||||
# print(f'Updated key: {unmatched_key}, model_dict[{unmatched_key}][:{loaded_dict_shape[0]}] = checkpoint["model_state_dict"][{unmatched_key}]')
|
||||
# elif len(loaded_dict_shape) >= 3:
|
||||
# raise ValueError(f'Unsupported shape: {loaded_dict_shape}')
|
||||
# model.load_state_dict(model_dict)
|
||||
|
||||
|
||||
|
||||
def save(self, epoch, model, optimizer):
|
||||
torch.save({
|
||||
'epoch': epoch,
|
||||
'model_state_dict': model.module.state_dict() if torch.cuda.device_count() > 1 else model.state_dict(),
|
||||
'optimizer_state_dict': optimizer.state_dict(),
|
||||
}, osp.join(self.output_dir, f'epoch-{epoch}.pth'))
|
||||
print(f'\nSave checkpoint to {osp.join(self.output_dir, f"epoch-{epoch}.pth")}')
|
||||
# osp.join(self.output_dir, f'{self.cfg.INPUT.MODALITY}_epoch-{epoch}.pth'))
|
||||
|
||||
def save_best(self, epoch, model, optimizer):
|
||||
# save_path = osp.join(self.output_dir, f'{self.cfg.INPUT.MODALITY}_best.pth')
|
||||
save_path = osp.join(self.output_dir, f'best.pth')
|
||||
|
||||
torch.save({
|
||||
'epoch': epoch,
|
||||
'model_state_dict': model.module.state_dict() if torch.cuda.device_count() > 1 else model.state_dict(),
|
||||
'optimizer_state_dict': optimizer.state_dict(),
|
||||
}, save_path)
|
||||
print(f'\nSave checkpoint to {save_path}')
|
||||
|
||||
def _load_checkpoint(self, checkpoint):
|
||||
if checkpoint is not None and osp.isfile(checkpoint):
|
||||
return torch.load(checkpoint, map_location=torch.device('cpu'))
|
||||
elif osp.isdir(checkpoint):
|
||||
import pdb; pdb.set_trace()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def setup_checkpointer(cfg, phase):
|
||||
return Checkpointer(cfg, phase)
|
||||
20
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/ek_utils.py
vendored
Normal file
20
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/ek_utils.py
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
import os.path as osp
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def action_to_verb_map(path_to_data,
|
||||
action_offset=False,
|
||||
verb_offset=False):
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'))
|
||||
a_to_v = {a[1]['id'] + action_offset: a[1]['verb'] + verb_offset
|
||||
for a in actions.iterrows()}
|
||||
return a_to_v
|
||||
|
||||
|
||||
def action_to_noun_map(path_to_data,
|
||||
action_offset=False,
|
||||
noun_offset=False):
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'))
|
||||
a_to_n = {a[1]['id'] + action_offset: a[1]['noun'] + noun_offset
|
||||
for a in actions.iterrows()}
|
||||
return a_to_n
|
||||
32
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/env.py
vendored
Normal file
32
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/env.py
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
__all__ = [
|
||||
'setup_random_seed',
|
||||
'setup_environment',
|
||||
]
|
||||
|
||||
import os
|
||||
import random
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
|
||||
def setup_random_seed(seed):
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed(seed)
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
torch.backends.cudnn.benchmark = False
|
||||
torch.backends.cudnn.deterministic = True
|
||||
|
||||
|
||||
def setup_environment(cfg):
|
||||
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.GPU
|
||||
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
if cfg.SEED is not None:
|
||||
setup_random_seed(cfg.SEED)
|
||||
return device
|
||||
32
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/logger.py
vendored
Normal file
32
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/logger.py
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
__all__ = ['setup_logger']
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import pprint
|
||||
|
||||
|
||||
def setup_logger(cfg, phase, quiet=False):
|
||||
logger = logging.getLogger('rekognition')
|
||||
level = logging.DEBUG if cfg.VERBOSE else logging.INFO
|
||||
logger.setLevel(level)
|
||||
ch = logging.StreamHandler(stream=sys.stdout)
|
||||
ch.setLevel(level)
|
||||
logger.addHandler(ch)
|
||||
|
||||
if phase == 'train':
|
||||
log_file = os.path.join(cfg.OUTPUT_DIR, 'log.txt')
|
||||
os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
|
||||
else:
|
||||
log_file = os.path.splitext(cfg.MODEL.CHECKPOINT)[0] + '.txt'
|
||||
fh = logging.FileHandler(log_file)
|
||||
fh.setLevel(logging.INFO)
|
||||
logger.addHandler(fh)
|
||||
|
||||
if not quiet:
|
||||
logger.info(pprint.pformat(cfg))
|
||||
|
||||
return logger
|
||||
114
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/parser.py
vendored
Normal file
114
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/parser.py
vendored
Normal file
@@ -0,0 +1,114 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os.path as osp
|
||||
|
||||
from rekognition_online_action_detection.config.defaults import get_cfg
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Rekognition Face Anti-Spoofing')
|
||||
parser.add_argument(
|
||||
'--config_file',
|
||||
default='',
|
||||
type=str,
|
||||
help='path to yaml config file',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--gpu',
|
||||
default='0',
|
||||
type=str,
|
||||
help='specify visible devices'
|
||||
)
|
||||
parser.add_argument(
|
||||
'opts',
|
||||
default=None,
|
||||
nargs='*',
|
||||
help='modify config options using the command-line',
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def assert_and_infer_cfg(cfg, args):
|
||||
# Setup the visible devices
|
||||
cfg.GPU = args.gpu
|
||||
|
||||
# Infer data info
|
||||
with open(cfg.DATA.DATA_INFO, 'r') as f:
|
||||
data_info = json.load(f)[cfg.DATA.DATA_NAME]
|
||||
|
||||
cfg.DATA.DATA_ROOT = data_info['data_root'] if cfg.DATA.DATA_ROOT is None else cfg.DATA.DATA_ROOT
|
||||
cfg.DATA.CLASS_NAMES = data_info['class_names'] if cfg.DATA.CLASS_NAMES is None else cfg.DATA.CLASS_NAMES
|
||||
cfg.DATA.NUM_CLASSES = data_info['num_classes'] if cfg.DATA.NUM_CLASSES is None else cfg.DATA.NUM_CLASSES
|
||||
cfg.DATA.IGNORE_INDEX = data_info['ignore_index'] if cfg.DATA.IGNORE_INDEX is None else cfg.DATA.IGNORE_INDEX
|
||||
cfg.DATA.METRICS = data_info['metrics'] if cfg.DATA.METRICS is None else cfg.DATA.METRICS
|
||||
cfg.DATA.FPS = data_info['fps'] if cfg.DATA.FPS is None else cfg.DATA.FPS
|
||||
cfg.DATA.TRAIN_SESSION_SET = data_info['train_session_set'] if cfg.DATA.TRAIN_SESSION_SET is None else cfg.DATA.TRAIN_SESSION_SET
|
||||
cfg.DATA.TEST_SESSION_SET = data_info['test_session_set'] if cfg.DATA.TEST_SESSION_SET is None else cfg.DATA.TEST_SESSION_SET
|
||||
|
||||
# Ignore two mis-labeled videos
|
||||
if False and cfg.DATA_NAME == 'THUMOS':
|
||||
cfg.DATA.TEST_SESSION_SET.remove('video_test_0000270')
|
||||
cfg.DATA.TEST_SESSION_SET.remove('video_test_0001496')
|
||||
|
||||
# Input assertions
|
||||
if cfg.INPUT.MODALITY == 'twostream':
|
||||
cfg.INPUT.MODALITY = 'visual+motion'
|
||||
if cfg.INPUT.MODALITY == 'threestream':
|
||||
cfg.INPUT.MODALITY = 'visual+motion+object'
|
||||
assert cfg.INPUT.MODALITY in ['visual', 'motion', 'object',
|
||||
'visual+motion', 'motion+object', 'visual+object',
|
||||
'visual+motion+object']
|
||||
|
||||
# Infer memory
|
||||
if cfg.MODEL.MODEL_NAME in ['LSTR']:
|
||||
cfg.MODEL.LSTR.AGES_MEMORY_LENGTH = cfg.MODEL.LSTR.AGES_MEMORY_SECONDS * cfg.DATA.FPS
|
||||
cfg.MODEL.LSTR.LONG_MEMORY_LENGTH = cfg.MODEL.LSTR.LONG_MEMORY_SECONDS * cfg.DATA.FPS
|
||||
cfg.MODEL.LSTR.WORK_MEMORY_LENGTH = cfg.MODEL.LSTR.WORK_MEMORY_SECONDS * cfg.DATA.FPS
|
||||
cfg.MODEL.LSTR.TOTAL_MEMORY_LENGTH = \
|
||||
cfg.MODEL.LSTR.AGES_MEMORY_LENGTH + \
|
||||
cfg.MODEL.LSTR.LONG_MEMORY_LENGTH + \
|
||||
cfg.MODEL.LSTR.WORK_MEMORY_LENGTH
|
||||
cfg.MODEL.LSTR.ANTICIPATION_LENGTH = cfg.MODEL.LSTR.ANTICIPATION_SECONDS * cfg.DATA.FPS
|
||||
assert cfg.MODEL.LSTR.AGES_MEMORY_LENGTH % cfg.MODEL.LSTR.AGES_MEMORY_SAMPLE_RATE == 0
|
||||
assert cfg.MODEL.LSTR.LONG_MEMORY_LENGTH % cfg.MODEL.LSTR.LONG_MEMORY_SAMPLE_RATE == 0
|
||||
assert cfg.MODEL.LSTR.WORK_MEMORY_LENGTH % cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE == 0
|
||||
assert cfg.MODEL.LSTR.ANTICIPATION_LENGTH % cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE == 0
|
||||
cfg.MODEL.LSTR.AGES_MEMORY_NUM_SAMPLES = cfg.MODEL.LSTR.AGES_MEMORY_LENGTH // cfg.MODEL.LSTR.AGES_MEMORY_SAMPLE_RATE
|
||||
cfg.MODEL.LSTR.LONG_MEMORY_NUM_SAMPLES = cfg.MODEL.LSTR.LONG_MEMORY_LENGTH // cfg.MODEL.LSTR.LONG_MEMORY_SAMPLE_RATE
|
||||
cfg.MODEL.LSTR.WORK_MEMORY_NUM_SAMPLES = cfg.MODEL.LSTR.WORK_MEMORY_LENGTH // cfg.MODEL.LSTR.WORK_MEMORY_SAMPLE_RATE
|
||||
cfg.MODEL.LSTR.TOTAL_MEMORY_NUM_SAMPLES = \
|
||||
cfg.MODEL.LSTR.AGES_MEMORY_NUM_SAMPLES + \
|
||||
cfg.MODEL.LSTR.LONG_MEMORY_NUM_SAMPLES + \
|
||||
cfg.MODEL.LSTR.WORK_MEMORY_NUM_SAMPLES
|
||||
cfg.MODEL.LSTR.ANTICIPATION_NUM_SAMPLES = cfg.MODEL.LSTR.ANTICIPATION_LENGTH // cfg.MODEL.LSTR.ANTICIPATION_SAMPLE_RATE
|
||||
|
||||
assert cfg.MODEL.LSTR.INFERENCE_MODE in ['batch', 'stream']
|
||||
|
||||
# Infer output dir
|
||||
config_name = osp.splitext(args.config_file)[0].split('/')[1:]
|
||||
cfg.OUTPUT_DIR = osp.join(cfg.OUTPUT_DIR, *config_name)
|
||||
if cfg.SESSION:
|
||||
cfg.OUTPUT_DIR = osp.join(cfg.OUTPUT_DIR, cfg.SESSION)
|
||||
|
||||
|
||||
def load_cfg():
|
||||
args = parse_args()
|
||||
cfg = get_cfg()
|
||||
|
||||
for i in range(0, len(args.opts), 2):
|
||||
key = args.opts[i]
|
||||
value = args.opts[i+1]
|
||||
if key == 'MODEL.LSTR.LONG_MEMORY_SECONDS' or key == 'MODEL.LSTR.WORK_MEMORY_SECONDS':
|
||||
args.opts[i+1] = int(value)
|
||||
print(f'(key, value): ({key}, {value}) is converted to integer type')
|
||||
|
||||
if args.config_file is not None:
|
||||
cfg.merge_from_file(args.config_file)
|
||||
if args.opts is not None:
|
||||
cfg.merge_from_list(args.opts)
|
||||
assert_and_infer_cfg(cfg, args)
|
||||
# config_file is in cfg.OUTPUT_DIR or args.config_file
|
||||
return cfg
|
||||
23
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/registry.py
vendored
Normal file
23
third_party/EgoSpeak/LSTR/src/rekognition_online_action_detection/utils/registry.py
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
def _register_generic(module_dict, module_name, module):
|
||||
assert module_name not in module_dict
|
||||
module_dict[module_name] = module
|
||||
|
||||
|
||||
class Registry(dict):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(Registry, self).__init__(*args, **kwargs)
|
||||
|
||||
def register(self, module_name, module=None):
|
||||
if module is not None:
|
||||
_register_generic(self, module_name, module)
|
||||
return
|
||||
|
||||
def register_fn(fn):
|
||||
_register_generic(self, module_name, fn)
|
||||
return fn
|
||||
|
||||
return register_fn
|
||||
162
third_party/EgoSpeak/LSTR/tools/eval/eval_fusion.py
vendored
Normal file
162
third_party/EgoSpeak/LSTR/tools/eval/eval_fusion.py
vendored
Normal file
@@ -0,0 +1,162 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
import pickle as pkl
|
||||
import pandas as pd
|
||||
import os.path as osp
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
from external.rulstm.RULSTM.utils import (get_marginal_indexes, marginalize, softmax,
|
||||
topk_accuracy_multiple_timesteps,
|
||||
topk_recall_multiple_timesteps,
|
||||
tta)
|
||||
|
||||
|
||||
def eval_fusion(pred_scores_files, weights):
|
||||
### Comments: Per-frame scores cost too much space...
|
||||
# perframe_gt_targets_fusion = {}
|
||||
# perframe_pred_scores_fusion = {}
|
||||
# for weight, pred_scores_file in zip(weights, pred_scores_files):
|
||||
# pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
# cfg = pred_scores['cfg']
|
||||
# assert cfg.DATA.DATA_NAME in ['EK55', 'EK100']
|
||||
# perframe_gt_targets = pred_scores['perframe_gt_targets']
|
||||
# perframe_pred_scores = pred_scores['perframe_pred_scores']
|
||||
|
||||
# for session in perframe_pred_scores:
|
||||
# if session not in perframe_pred_scores_fusion:
|
||||
# perframe_pred_scores_fusion[session] = weight * perframe_pred_scores[session]
|
||||
# else:
|
||||
# perframe_pred_scores_fusion[session] += weight * perframe_pred_scores[session]
|
||||
# perframe_gt_targets_fusion[session] = perframe_gt_targets[session]
|
||||
|
||||
# # Compute results
|
||||
# result = compute_result['perframe'](
|
||||
# cfg,
|
||||
# np.concatenate(list(perframe_gt_targets_fusion.values()), axis=0),
|
||||
# np.concatenate(list(perframe_pred_scores_fusion.values()), axis=0),
|
||||
# )
|
||||
# logging.info('Action detection perframe m{}: {:.5f}'.format(
|
||||
# cfg.DATA.METRICS, result['mean_AP']
|
||||
# ))
|
||||
|
||||
action_scores = None
|
||||
for weight, pred_scores_file in zip(weights, pred_scores_files):
|
||||
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
cfg = pred_scores['cfg']
|
||||
assert cfg.DATA.DATA_NAME in ['EK55', 'EK100']
|
||||
if action_scores is None:
|
||||
action_scores = weight * pred_scores['action_scores']
|
||||
else:
|
||||
action_scores += weight * pred_scores['action_scores']
|
||||
# segment-level evaluation on EK55/EK100
|
||||
if cfg.DATA.DATA_NAME == 'EK55':
|
||||
path_to_data = 'external/rulstm/RULSTM/data/ek55/'
|
||||
segment_list = pd.read_csv(osp.join(path_to_data, 'validation.csv'),
|
||||
names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
||||
skipinitialspace=True)
|
||||
predictions, labels, ids = [], [], []
|
||||
discarded_labels, discarded_ids = [], []
|
||||
for idx in segment_list['id']:
|
||||
segment = segment_list[segment_list['id'] == idx]
|
||||
start_step = int(np.floor(segment['start_f'].values.astype(int) / 30 * cfg.DATA.FPS))
|
||||
vid = segment['video'].values[0].strip()
|
||||
# preds = (perframe_pred_scores_fusion[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
if start_step >= cfg.MODEL.LSTR.ANTICIPATION_LENGTH:
|
||||
# predictions.append(preds)
|
||||
labels.append(segment[['verb', 'noun', 'action']].values.astype(int))
|
||||
ids.append(idx)
|
||||
else:
|
||||
discarded_labels.append(segment[['verb', 'noun', 'action']].values.astype(int))
|
||||
discarded_ids.append(idx)
|
||||
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'), index_col='id')
|
||||
|
||||
# action_scores = np.concatenate(predictions)
|
||||
labels = np.concatenate(labels)
|
||||
ids = np.array(ids)
|
||||
|
||||
vi = get_marginal_indexes(actions, 'verb')
|
||||
ni = get_marginal_indexes(actions, 'noun')
|
||||
|
||||
action_probs = softmax(action_scores.reshape(-1, action_scores.shape[-1]))
|
||||
# action_probs = action_scores.reshape(-1, action_scores.shape[-1])
|
||||
# action_probs = action_probs / action_probs.sum(-1, keepdims=True)
|
||||
print(action_probs.shape)
|
||||
verb_scores = marginalize(action_probs, vi).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
noun_scores = marginalize(action_probs, ni).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
|
||||
include_discarded = (cfg.DATA.DATA_NAME == 'EK100')
|
||||
if include_discarded:
|
||||
dlab = np.array(discarded_labels)
|
||||
dislab = np.array(discarded_ids)
|
||||
ids = np.concatenate([ids, dislab])
|
||||
num_disc = len(dlab)
|
||||
labels = np.concatenate([labels, dlab])
|
||||
verb_scores = np.concatenate((verb_scores, np.zeros((num_disc, *verb_scores.shape[1:]))))
|
||||
noun_scores = np.concatenate((noun_scores, np.zeros((num_disc, *noun_scores.shape[1:]))))
|
||||
action_scores = np.concatenate((action_scores, np.zeros((num_disc, *action_scores.shape[1:]))))
|
||||
|
||||
verb_labels, noun_labels, action_labels = labels[:, 0], labels[:, 1], labels[:, 2]
|
||||
verb_accuracies = topk_accuracy_multiple_timesteps(verb_scores, verb_labels)
|
||||
noun_accuracies = topk_accuracy_multiple_timesteps(noun_scores, noun_labels)
|
||||
action_accuracies = topk_accuracy_multiple_timesteps(action_scores, action_labels)
|
||||
|
||||
many_shot_verbs = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_verbs.csv'))['verb_class'].values
|
||||
many_shot_nouns = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_nouns.csv'))['noun_class'].values
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'))
|
||||
a_to_vn = {a[1]['id']: tuple(a[1][['verb', 'noun']].values)
|
||||
for a in actions.iterrows()}
|
||||
many_shot_actions = [a for a, (v, n) in a_to_vn.items()
|
||||
if v in many_shot_verbs or n in many_shot_nouns]
|
||||
|
||||
verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels,
|
||||
k=5, classes=many_shot_verbs)
|
||||
noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels,
|
||||
k=5, classes=many_shot_nouns)
|
||||
action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels,
|
||||
k=5, classes=many_shot_actions)
|
||||
|
||||
all_accuracies = np.concatenate([verb_accuracies, noun_accuracies, action_accuracies,
|
||||
verb_recalls, noun_recalls, action_recalls])
|
||||
all_accuracies = all_accuracies[[0, 1, 6, 2, 3, 7, 4, 5, 8]]
|
||||
indices = [
|
||||
('Verb', 'Top-1 Accuracy'), ('Verb', 'Top-5 Accuracy'),
|
||||
('Verb', 'Mean Top-5 Recall'),
|
||||
('Noun', 'Top-1 Accuracy'), ('Noun', 'Top-5 Accuracy'),
|
||||
('Noun', 'Mean Top-5 Recall'),
|
||||
('Action', 'Top-1 Accuracy'), ('Action', 'Top-5 Accuracy'),
|
||||
('Action', 'Mean Top-5 Recall'),
|
||||
]
|
||||
cc = np.linspace(1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_LENGTH,
|
||||
1 / cfg.DATA.FPS, cfg.MODEL.LSTR.ANTICIPATION_LENGTH, dtype=str)
|
||||
scores = pd.DataFrame(all_accuracies * 100, columns=cc,
|
||||
index=pd.MultiIndex.from_tuples(indices))
|
||||
print(scores)
|
||||
|
||||
tta_verb = tta(verb_scores, verb_labels)
|
||||
tta_noun = tta(noun_scores, noun_labels)
|
||||
tta_action = tta(action_scores, action_labels)
|
||||
print(f'\nMean TtA(5): VERB {tta_verb:0.2f} '
|
||||
f'NOUN: {tta_noun:0.2f} ACTION: {tta_action:0.2f}')
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pred_scores_files', nargs='+', required=True)
|
||||
parser.add_argument('--weights', type=float, nargs='+', required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
assert len(args.pred_scores_files) == len(args.weights)
|
||||
eval_fusion(args.pred_scores_files, args.weights)
|
||||
36
third_party/EgoSpeak/LSTR/tools/eval/eval_perframe.py
vendored
Normal file
36
third_party/EgoSpeak/LSTR/tools/eval/eval_perframe.py
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
import pickle as pkl
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
|
||||
|
||||
def eval_perframe(pred_scores_file):
|
||||
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
cfg = pred_scores['cfg']
|
||||
perframe_gt_targets = pred_scores['perframe_gt_targets']
|
||||
perframe_pred_scores = pred_scores['perframe_pred_scores']
|
||||
|
||||
# Compute results
|
||||
result = compute_result['perframe'](
|
||||
cfg,
|
||||
np.concatenate(list(perframe_gt_targets.values()), axis=0),
|
||||
np.concatenate(list(perframe_pred_scores.values()), axis=0),
|
||||
)
|
||||
logging.info('Action detection perframe m{}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, result['mean_AP']
|
||||
))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pred_scores_file', type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
eval_perframe(args.pred_scores_file)
|
||||
37
third_party/EgoSpeak/LSTR/tools/eval/eval_perstage.py
vendored
Normal file
37
third_party/EgoSpeak/LSTR/tools/eval/eval_perstage.py
vendored
Normal file
@@ -0,0 +1,37 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
import pickle as pkl
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
|
||||
|
||||
def eval_perstage(pred_scores_file):
|
||||
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
cfg = pred_scores['cfg']
|
||||
perframe_gt_targets = pred_scores['perframe_gt_targets']
|
||||
perframe_pred_scores = pred_scores['perframe_pred_scores']
|
||||
|
||||
# Compute results
|
||||
result = compute_result['perstage'](
|
||||
cfg,
|
||||
np.concatenate(list(perframe_gt_targets.values()), axis=0),
|
||||
np.concatenate(list(perframe_pred_scores.values()), axis=0),
|
||||
)
|
||||
for stage_name in result:
|
||||
logging.info('Perframe m{} of stage {}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, stage_name, result[stage_name]['mean_AP']
|
||||
))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pred_scores_file', type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
eval_perstage(args.pred_scores_file)
|
||||
80
third_party/EgoSpeak/LSTR/tools/generate_targets_epic_kitchen.py
vendored
Normal file
80
third_party/EgoSpeak/LSTR/tools/generate_targets_epic_kitchen.py
vendored
Normal file
@@ -0,0 +1,80 @@
|
||||
import argparse
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import os.path as osp
|
||||
|
||||
|
||||
def str2sec(string):
|
||||
hh, mm, ss = string.split(':')
|
||||
return float(hh) * 3600 + float(mm) * 60 + float(ss)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Generate targets')
|
||||
parser.add_argument('--dataset', type=str,
|
||||
choices=['EK55', 'EK100'], default='EK55')
|
||||
parser.add_argument('--data-path', type=str,
|
||||
default='/scratch/cluster/yzhao/Datasets/EPIC-KITCHENS/')
|
||||
parser.add_argument('--targets-path', type=str,
|
||||
default='/scratch/cluster/yzhao/Datasets/'
|
||||
'EPIC-KITCHENS/features/target_perframe/')
|
||||
parser.add_argument('--fps', type=int, default=30)
|
||||
parser.add_argument('--tick', type=float, default=7.5)
|
||||
parser.add_argument('--anno-path', type=str,
|
||||
default='external/rulstm/RULSTM/data/ek55/')
|
||||
parser.add_argument('--class-type', type=str, default='action',
|
||||
choices=['verb', 'noun', 'action'])
|
||||
args = parser.parse_args()
|
||||
|
||||
# initialize the targets
|
||||
targets = {}
|
||||
if args.class_type == 'action':
|
||||
num_classes = 2513 + 1
|
||||
elif args.class_type == 'verb':
|
||||
actions = pd.read_csv(osp.join(args.anno_path, 'actions.csv'))
|
||||
a_to_v = {a[1]['id'] + 1: a[1]['verb'] + 1
|
||||
for a in actions.iterrows()}
|
||||
num_classes = max(a_to_v.values()) + 1
|
||||
elif args.class_type == 'noun':
|
||||
actions = pd.read_csv(osp.join(args.anno_path, 'actions.csv'))
|
||||
a_to_n = {a[1]['id'] + 1: a[1]['noun'] + 1
|
||||
for a in actions.iterrows()}
|
||||
num_classes = max(a_to_n.values()) + 1
|
||||
|
||||
for subset in ['training', 'validation']:
|
||||
vids = [line.strip() for line in open('{}/{}_videos.csv'.format(args.anno_path, subset))]
|
||||
anno_per_video = {}
|
||||
for line in open('{}/{}.csv'.format(args.anno_path, subset)):
|
||||
idx, vid, f_start, f_end, c_verb, c_noun, c_verbnoun = line.strip().split(',')
|
||||
vid = vid.strip()
|
||||
if vid not in anno_per_video:
|
||||
anno_per_video[vid] = [(int(f_start), int(f_end), int(c_verb), int(c_noun), int(c_verbnoun))]
|
||||
else:
|
||||
anno_per_video[vid].append((int(f_start), int(f_end), int(c_verb), int(c_noun), int(c_verbnoun)))
|
||||
for vid in vids:
|
||||
feature = np.load('{}/features/rgb_kinetics_resnet50/{}.npy'.format(args.data_path, vid))
|
||||
feature_len = feature.shape[0]
|
||||
targets[vid] = np.zeros((feature_len,
|
||||
num_classes))
|
||||
targets[vid][:, 0] = 1
|
||||
for anno in anno_per_video[vid]:
|
||||
print(vid, anno)
|
||||
ss = int(np.floor(anno[0] / args.tick))
|
||||
ee = min(int(np.ceil(anno[1] / args.tick)), feature_len)
|
||||
if args.class_type == 'action':
|
||||
ci = anno[4]
|
||||
elif args.class_type == 'verb':
|
||||
ci = anno[2]
|
||||
elif args.class_type == 'noun':
|
||||
ci = anno[3]
|
||||
for si in range(ss, ee):
|
||||
targets[vid][si, ci + 1] = 1
|
||||
targets[vid][si, 0] = 0
|
||||
|
||||
print('Dumping npy files..')
|
||||
for vid in targets:
|
||||
np.save(args.targets_path + '{}.npy'.format(vid), targets[vid])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
45
third_party/EgoSpeak/LSTR/tools/test_net.py
vendored
Normal file
45
third_party/EgoSpeak/LSTR/tools/test_net.py
vendored
Normal file
@@ -0,0 +1,45 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
|
||||
from rekognition_online_action_detection.engines import do_inference
|
||||
from rekognition_online_action_detection.models import build_model
|
||||
from rekognition_online_action_detection.utils.checkpointer import \
|
||||
setup_checkpointer
|
||||
from rekognition_online_action_detection.utils.env import setup_environment
|
||||
from rekognition_online_action_detection.utils.logger import setup_logger
|
||||
from rekognition_online_action_detection.utils.parser import load_cfg
|
||||
from utils import validate_cfg
|
||||
|
||||
|
||||
def main(cfg):
|
||||
validate_cfg(cfg)
|
||||
config_name = os.path.basename(cfg.OUTPUT_DIR)
|
||||
|
||||
print(f"cfg.MODEL.CHECKPOINT: {cfg.MODEL.CHECKPOINT}")
|
||||
print(f"cfg.OUTPUT_DIR: {cfg.OUTPUT_DIR}")
|
||||
|
||||
# Setup configurations
|
||||
device = setup_environment(cfg)
|
||||
checkpointer = setup_checkpointer(cfg, phase='test')
|
||||
logger = setup_logger(cfg, phase='test')
|
||||
|
||||
print(cfg)
|
||||
print(f"DATA.IGNORE_INDEX: {cfg.DATA.IGNORE_INDEX}")
|
||||
# Build model
|
||||
model = build_model(cfg, device)
|
||||
|
||||
# Load pretrained model
|
||||
checkpointer.load(model)
|
||||
|
||||
do_inference(
|
||||
cfg,
|
||||
model,
|
||||
device,
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(load_cfg())
|
||||
67
third_party/EgoSpeak/LSTR/tools/train_net.py
vendored
Normal file
67
third_party/EgoSpeak/LSTR/tools/train_net.py
vendored
Normal file
@@ -0,0 +1,67 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import wandb
|
||||
from rekognition_online_action_detection.criterions import build_criterion
|
||||
from rekognition_online_action_detection.datasets import build_data_loader
|
||||
from rekognition_online_action_detection.engines import do_train
|
||||
from rekognition_online_action_detection.models import build_model
|
||||
from rekognition_online_action_detection.optimizers import (build_optimizer,
|
||||
build_scheduler)
|
||||
from rekognition_online_action_detection.utils.checkpointer import \
|
||||
setup_checkpointer
|
||||
from rekognition_online_action_detection.utils.env import setup_environment
|
||||
from rekognition_online_action_detection.utils.logger import setup_logger
|
||||
from rekognition_online_action_detection.utils.parser import load_cfg
|
||||
from utils import parse_config
|
||||
|
||||
|
||||
def main(cfg):
|
||||
parse_config(cfg)
|
||||
# Setup configurations
|
||||
device = setup_environment(cfg)
|
||||
checkpointer = setup_checkpointer(cfg, phase="train")
|
||||
logger = setup_logger(cfg, phase="train")
|
||||
|
||||
print(cfg)
|
||||
print(f"cfg['SOLVER']['NUM_EPOCHS']: {cfg['SOLVER']['NUM_EPOCHS']}")
|
||||
print(f"cfg['MODEL']['CHECKPOINT']: {cfg['MODEL']['CHECKPOINT']}")
|
||||
|
||||
# Build data loaders
|
||||
data_loaders = {phase: build_data_loader(cfg, phase) for phase in cfg.SOLVER.PHASES}
|
||||
|
||||
# Build model
|
||||
model = build_model(cfg, device)
|
||||
|
||||
# Build criterion
|
||||
criterion = build_criterion(cfg, device)
|
||||
|
||||
# Build optimizer
|
||||
optimizer = build_optimizer(cfg, model)
|
||||
|
||||
# Load pretrained model and optimizer
|
||||
checkpointer.load(model, optimizer)
|
||||
|
||||
# Build scheduler
|
||||
scheduler = build_scheduler(cfg, optimizer, len(data_loaders["train"]))
|
||||
|
||||
do_train(
|
||||
cfg,
|
||||
data_loaders,
|
||||
model,
|
||||
criterion,
|
||||
optimizer,
|
||||
scheduler,
|
||||
device,
|
||||
checkpointer,
|
||||
logger,
|
||||
)
|
||||
|
||||
wandb.finish() if cfg.USE_WANDB else None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(load_cfg())
|
||||
97
third_party/EgoSpeak/LSTR/tools/utils.py
vendored
Normal file
97
third_party/EgoSpeak/LSTR/tools/utils.py
vendored
Normal file
@@ -0,0 +1,97 @@
|
||||
import os
|
||||
|
||||
import wandb
|
||||
|
||||
|
||||
def parse_config(cfg):
|
||||
modality = get_modality(cfg)
|
||||
|
||||
cfg_filename = os.path.basename(cfg.OUTPUT_DIR)
|
||||
|
||||
os.environ["WANDB__SERVICE_WAIT"] = "3000"
|
||||
cfg_filename_abbr = cfg_filename.split('_1x_')[-1]
|
||||
wandb_postfix = generate_wandb_postfix(cfg, modality)
|
||||
scheduler_name = cfg.SOLVER.SCHEDULER.SCHEDULER_NAME
|
||||
learning_rate = cfg.SOLVER.BASE_LR
|
||||
weight_decay = cfg.SOLVER.WEIGHT_DECAY
|
||||
|
||||
cfg_filename = cfg_filename.replace('long_512', f'long_{cfg.MODEL.LSTR.LONG_MEMORY_SECONDS}')
|
||||
cfg_filename = cfg_filename.replace('work_8', f'work_{cfg.MODEL.LSTR.WORK_MEMORY_SECONDS}')
|
||||
|
||||
cfg.OUTPUT_DIR = os.path.join(os.path.dirname(cfg.OUTPUT_DIR), cfg_filename)
|
||||
|
||||
if cfg.SAVE_HPARAM:
|
||||
# save different dir containing hparam
|
||||
hparam_info = '_'.join([f"lr_{learning_rate}", f"wd_{weight_decay}", f"schd_{scheduler_name}"])
|
||||
cfg.OUTPUT_DIR = os.path.join(cfg.OUTPUT_DIR, hparam_info, modality)
|
||||
print(f"HPARAM: {hparam_info}")
|
||||
print(f"cfg.OUTPUT_DIR: {cfg.OUTPUT_DIR}")
|
||||
|
||||
wandb_name = cfg_filename_abbr + '_' + wandb_postfix
|
||||
|
||||
if cfg.SAVE_HPARAM:
|
||||
wandb_name += '_' + '_'.join([f"lr_{learning_rate}", f"wd_{weight_decay}", f"schd_{scheduler_name}"])
|
||||
|
||||
if cfg.USE_WANDB:
|
||||
print("*** init wandb ***")
|
||||
print(f"wandb_name: {wandb_name}")
|
||||
|
||||
def get_modality(cfg):
|
||||
modality = ''
|
||||
if 'motion' in cfg.INPUT.MODALITY:
|
||||
modality += 'A'
|
||||
if 'visual' in cfg.INPUT.MODALITY:
|
||||
modality += 'V'
|
||||
if 'object' in cfg.INPUT.MODALITY:
|
||||
modality += 'T'
|
||||
elif cfg['INPUT']['MODALITY'] == 'twostream':
|
||||
modality = 'AV'
|
||||
|
||||
if modality == '':
|
||||
raise ValueError(f"Unknown modality: {cfg['INPUT']['MODALITY']}")
|
||||
|
||||
return modality
|
||||
|
||||
|
||||
def generate_wandb_postfix(cfg, modality):
|
||||
visual_feature_type = '_'.join(cfg['INPUT']['VISUAL_FEATURE'].split('_')[1:])
|
||||
audio_feature_type = '_'.join(cfg['INPUT']['MOTION_FEATURE'].split('_')[1:])
|
||||
text_feature_type = '_'.join(cfg['INPUT']['OBJECT_FEATURE'].split('_')[1:])
|
||||
|
||||
if cfg.MODEL.LSTR.LONG_MEMORY_USE_PE == True:
|
||||
model_name = 'LSTR_TESTRA'
|
||||
else:
|
||||
model_name = 'TESTRA'
|
||||
|
||||
if '_aa' in cfg.OUTPUT_DIR:
|
||||
model_name += '_aa'
|
||||
|
||||
name_parts = [
|
||||
model_name,
|
||||
cfg['DATA']['DATA_NAME'],
|
||||
modality,
|
||||
# visual_feature_type,
|
||||
# audio_feature_type,
|
||||
# text_feature_type,
|
||||
# f"epoch_{cfg['SOLVER']['NUM_EPOCHS']}"
|
||||
]
|
||||
if cfg['MODEL']['CHECKPOINT']:
|
||||
name_parts.append(f"ckpt_{os.path.basename(cfg['MODEL']['CHECKPOINT']).split('.')[0]}")
|
||||
return '_'.join(name_parts)
|
||||
|
||||
def get_wandb_name(cfg, cfg_filename, modality):
|
||||
cfg_filename_abbr = cfg_filename.split('_1x_')[-1]
|
||||
wandb_postfix = generate_wandb_postfix(cfg, modality)
|
||||
scheduler_name = cfg.SOLVER.SCHEDULER.SCHEDULER_NAME
|
||||
learning_rate = cfg.SOLVER.BASE_LR
|
||||
weight_decay = cfg.SOLVER.WEIGHT_DECAY
|
||||
wandb_name = cfg_filename_abbr + '_' + wandb_postfix + '_' + '_'.join([f"lr_{learning_rate}", f"wd_{weight_decay}", f"schd_{scheduler_name}"])
|
||||
return wandb_name
|
||||
|
||||
def validate_cfg(cfg):
|
||||
if cfg.DATA.NUM_CLASSES != len(cfg.DATA.CLASS_NAMES):
|
||||
print(f"len(cfg.DATA.CLASS_NAMES): {len(cfg.DATA.CLASS_NAMES)} is not equal to cfg.DATA.NUM_CLASSES: {cfg.DATA.NUM_CLASSES}")
|
||||
cfg.DATA.CLASS_NAMES = cfg.DATA.CLASS_NAMES[:cfg.DATA.NUM_CLASSES]
|
||||
print(f"Adjusted cfg.DATA.CLASS_NAMES: {cfg.DATA.CLASS_NAMES}")
|
||||
|
||||
|
||||
27
third_party/EgoSpeak/MiniROAD/.gitignore
vendored
Normal file
27
third_party/EgoSpeak/MiniROAD/.gitignore
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
||||
# PyTorch models
|
||||
*.pth
|
||||
|
||||
# Miscellaneous
|
||||
*.DS_Store
|
||||
*.swp
|
||||
*.pyc
|
||||
*.pkl
|
||||
|
||||
*.out
|
||||
|
||||
dataset/
|
||||
# Dataset soft links
|
||||
data/
|
||||
|
||||
# Output files
|
||||
output/
|
||||
output_*/
|
||||
|
||||
checkpoints/
|
||||
|
||||
notebooks/
|
||||
wandb/
|
||||
32
third_party/EgoSpeak/MiniROAD/configs/EasyCom/mamba_aa_easycom.yaml
vendored
Executable file
32
third_party/EgoSpeak/MiniROAD/configs/EasyCom/mamba_aa_easycom.yaml
vendored
Executable file
@@ -0,0 +1,32 @@
|
||||
model: 'MambaA'
|
||||
data_name: 'EasyCom_ANTICIPATION'
|
||||
task: 'ANTICIPATION'
|
||||
loss: 'ANTICIPATION' # NONUNIFORM, originally
|
||||
metric: 'AP' # AP or cAP
|
||||
optimizer: 'AdamW'
|
||||
device: 'cuda'
|
||||
feature_pretrained: 'resnet50'
|
||||
root_path: './data/EasyCom'
|
||||
rgb_type: 'rgb_kinetics_resnet50'
|
||||
flow_type: 'audio_wav2vec2'
|
||||
annotation_type: 'target_perframe'
|
||||
video_list_path: 'data_info/data_info.json'
|
||||
output_path: './output_cfg'
|
||||
window_size: 128
|
||||
batch_size: 64
|
||||
test_batch_size: 1
|
||||
# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
|
||||
num_epoch: 30
|
||||
lr: 0.0001
|
||||
weight_decay: 0.05
|
||||
num_workers: 4
|
||||
dropout: 0.20
|
||||
num_classes: 3
|
||||
embedding_dim: 2048
|
||||
hidden_dim: 1024
|
||||
num_layers: 1
|
||||
stride: 4
|
||||
anticipation_length: 10
|
||||
actionness: False
|
||||
ignore_index: -100
|
||||
seed: 20
|
||||
32
third_party/EgoSpeak/MiniROAD/configs/EasyCom/miniroad_aa_easycom.yaml
vendored
Executable file
32
third_party/EgoSpeak/MiniROAD/configs/EasyCom/miniroad_aa_easycom.yaml
vendored
Executable file
@@ -0,0 +1,32 @@
|
||||
model: 'MiniROADA'
|
||||
data_name: 'EasyCom_ANTICIPATION'
|
||||
task: 'ANTICIPATION'
|
||||
loss: 'ANTICIPATION' # NONUNIFORM, originally
|
||||
metric: 'AP' # AP or cAP
|
||||
optimizer: 'AdamW'
|
||||
device: 'cuda'
|
||||
feature_pretrained: 'resnet50'
|
||||
root_path: './data/EasyCom'
|
||||
rgb_type: 'rgb_kinetics_resnet50'
|
||||
flow_type: 'audio_wav2vec2'
|
||||
annotation_type: 'target_perframe'
|
||||
video_list_path: 'data_info/data_info.json'
|
||||
output_path: './output_cfg'
|
||||
window_size: 128
|
||||
batch_size: 64
|
||||
test_batch_size: 1
|
||||
# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
|
||||
num_epoch: 30
|
||||
lr: 0.0001
|
||||
weight_decay: 0.05
|
||||
num_workers: 4
|
||||
dropout: 0.20
|
||||
num_classes: 3 # [background, target speaker speaking, other person speaking]
|
||||
embedding_dim: 2048
|
||||
hidden_dim: 1024
|
||||
num_layers: 1
|
||||
stride: 4
|
||||
anticipation_length: 10
|
||||
actionness: False
|
||||
ignore_index: -100
|
||||
seed: 20
|
||||
32
third_party/EgoSpeak/MiniROAD/configs/YTConv/mamba_aa_ytconv.yaml
vendored
Executable file
32
third_party/EgoSpeak/MiniROAD/configs/YTConv/mamba_aa_ytconv.yaml
vendored
Executable file
@@ -0,0 +1,32 @@
|
||||
model: 'MambaA'
|
||||
data_name: 'YTConv_ANTICIPATION'
|
||||
data_info_name: 'shard_0'
|
||||
task: 'ANTICIPATION'
|
||||
loss: 'ANTICIPATION' # NONUNIFORM, originally
|
||||
metric: 'AP' # AP or cAP
|
||||
optimizer: 'AdamW'
|
||||
device: 'cuda'
|
||||
feature_pretrained: 'resnet50'
|
||||
root_path: './data/YTConv/shard_0'
|
||||
rgb_type: 'rgb_kinetics_resnet50'
|
||||
flow_type: 'audio_wav2vec2' # or 'none'?
|
||||
annotation_type: 'target_perframe/case1'
|
||||
video_list_path: 'data_info/YTConv_info.json'
|
||||
output_path: './output_cfg'
|
||||
window_size: 128
|
||||
batch_size: 32
|
||||
test_batch_size: 1
|
||||
# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
|
||||
num_epoch: 30
|
||||
lr: 0.0001
|
||||
weight_decay: 0.05
|
||||
num_workers: 4
|
||||
dropout: 0.20
|
||||
num_classes: 2 # for YTConv pretraining, there is no
|
||||
embedding_dim: 2048
|
||||
hidden_dim: 1024
|
||||
num_layers: 1
|
||||
stride: 4
|
||||
anticipation_length: 10
|
||||
actionness: False
|
||||
ignore_index: -100
|
||||
32
third_party/EgoSpeak/MiniROAD/configs/YTConv/miniroad_aa_ytconv.yaml
vendored
Normal file
32
third_party/EgoSpeak/MiniROAD/configs/YTConv/miniroad_aa_ytconv.yaml
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
model: 'MiniROADA'
|
||||
data_name: 'YTConv_ANTICIPATION'
|
||||
data_info_name: 'shard_0'
|
||||
task: 'ANTICIPATION'
|
||||
loss: 'ANTICIPATION' # NONUNIFORM, originally
|
||||
metric: 'AP' # AP or cAP
|
||||
optimizer: 'AdamW'
|
||||
device: 'cuda'
|
||||
feature_pretrained: 'resnet50'
|
||||
root_path: './data/YTConv/shard_0'
|
||||
rgb_type: 'rgb_kinetics_resnet50'
|
||||
flow_type: 'audio_wav2vec2'
|
||||
annotation_type: 'target_perframe'
|
||||
video_list_path: 'data_info/YTConv_info.json'
|
||||
output_path: './output_cfg'
|
||||
window_size: 128
|
||||
batch_size: 64
|
||||
test_batch_size: 1
|
||||
# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
|
||||
num_epoch: 30
|
||||
lr: 0.0001
|
||||
weight_decay: 0.05
|
||||
num_workers: 4
|
||||
dropout: 0.20
|
||||
num_classes: 2 # for pretraining, there is no target speaker
|
||||
embedding_dim: 2048
|
||||
hidden_dim: 1024
|
||||
num_layers: 1
|
||||
stride: 4
|
||||
anticipation_length: 10
|
||||
actionness: False
|
||||
ignore_index: -100
|
||||
0
third_party/EgoSpeak/MiniROAD/configs/__init__.py
vendored
Normal file
0
third_party/EgoSpeak/MiniROAD/configs/__init__.py
vendored
Normal file
36
third_party/EgoSpeak/MiniROAD/configs/ego4d/mamba_aa_ego4dshuffle.yaml
vendored
Executable file
36
third_party/EgoSpeak/MiniROAD/configs/ego4d/mamba_aa_ego4dshuffle.yaml
vendored
Executable file
@@ -0,0 +1,36 @@
|
||||
model: 'MambaA'
|
||||
data_name: 'EGO4D_ANTICIPATION'
|
||||
data_info_name: 'EGO4DSHUFFLE'
|
||||
task: 'ANTICIPATION'
|
||||
loss: 'ANTICIPATION' # NONUNIFORM, originally
|
||||
metric: 'AP' # AP or cAP
|
||||
optimizer: 'AdamW'
|
||||
device: 'cuda'
|
||||
feature_pretrained: 'resnet50'
|
||||
root_path: './data/EGO4D'
|
||||
rgb_type: 'rgb_kinetics_resnet50'
|
||||
flow_type: 'audio_wav2vec2'
|
||||
annotation_type: 'target_perframe'
|
||||
video_list_path: 'data_info/data_info.json'
|
||||
output_path: './output_cfg'
|
||||
window_size: 128
|
||||
batch_size: 64
|
||||
test_batch_size: 1
|
||||
# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
|
||||
num_epoch: 30
|
||||
lr: 0.0001
|
||||
weight_decay: 0.05
|
||||
num_workers: 4
|
||||
dropout: 0.20
|
||||
num_classes: 3 # including background
|
||||
embedding_dim: 2048
|
||||
hidden_dim: 1024
|
||||
d_state: 16
|
||||
d_conv: 4
|
||||
expand: 2
|
||||
num_layers: 1
|
||||
stride: 4
|
||||
anticipation_length: 10
|
||||
actionness: False
|
||||
ignore_index: -100
|
||||
seed: 20
|
||||
32
third_party/EgoSpeak/MiniROAD/configs/ego4d/miniroad_aa_ego4dshuffle.yaml
vendored
Executable file
32
third_party/EgoSpeak/MiniROAD/configs/ego4d/miniroad_aa_ego4dshuffle.yaml
vendored
Executable file
@@ -0,0 +1,32 @@
|
||||
model: 'MiniROADA'
|
||||
data_name: 'EGO4D_ANTICIPATION'
|
||||
data_info_name: 'EGO4DSHUFFLE'
|
||||
task: 'ANTICIPATION'
|
||||
loss: 'ANTICIPATION' # NONUNIFORM, originally
|
||||
metric: 'AP' # AP or cAP
|
||||
optimizer: 'AdamW'
|
||||
device: 'cuda'
|
||||
feature_pretrained: 'resnet50'
|
||||
root_path: './data/EGO4D'
|
||||
rgb_type: 'rgb_kinetics_resnet50'
|
||||
flow_type: 'audio_wav2vec2' # or 'none'?
|
||||
annotation_type: 'target_perframe'
|
||||
video_list_path: 'data_info/data_info.json'
|
||||
output_path: './output_cfg'
|
||||
window_size: 128
|
||||
batch_size: 64
|
||||
test_batch_size: 1
|
||||
# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
|
||||
num_epoch: 30
|
||||
lr: 0.0001
|
||||
weight_decay: 0.05
|
||||
num_workers: 4
|
||||
dropout: 0.20
|
||||
num_classes: 3
|
||||
embedding_dim: 2048
|
||||
hidden_dim: 1024
|
||||
num_layers: 1
|
||||
stride: 4
|
||||
anticipation_length: 10
|
||||
actionness: False
|
||||
ignore_index: -100
|
||||
26
third_party/EgoSpeak/MiniROAD/configs/feature_sizes.py
vendored
Normal file
26
third_party/EgoSpeak/MiniROAD/configs/feature_sizes.py
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
# dimension of features extracted from different models
|
||||
# used in feature_head, dataset, models
|
||||
|
||||
FEATURE_SIZES = {
|
||||
'rgb_anet_resnet50': 2048,
|
||||
'flow_anet_resnet50': 2048,
|
||||
'rgb_kinetics_bninception': 1024,
|
||||
'flow_kinetics_bninception': 1024,
|
||||
'rgb_farl': 512,
|
||||
'rgb_kinetics_resnet50': 2048,
|
||||
'rgb_kinetics_resnet50_debug': 2048,
|
||||
'rgb_kinetics_resnet50_debug_resized': 2048,
|
||||
'flow_kinetics_resnet50': 2048,
|
||||
'flow_nv_kinetics_bninception': 1024,
|
||||
'rgb_kinetics_i3d': 2048,
|
||||
'flow_kinetics_i3d': 2048,
|
||||
'audio_wav2vec2': 5120,
|
||||
'audio_wav2vec2_left': 5120,
|
||||
'audio_wav2vec2_right': 5120,
|
||||
'text_word2vec': 300,
|
||||
'text_fasttext_crawl': 300,
|
||||
'text_gpt2_last_subword_window_size_0': 768,
|
||||
'text_gpt2_last_subword_window_size_20': 768,
|
||||
'text_gpt2_last_subword_window_size_50': 768,
|
||||
'text_gpt2_last_subword_window_size_100': 768,
|
||||
}
|
||||
2
third_party/EgoSpeak/MiniROAD/criterions/__init__.py
vendored
Executable file
2
third_party/EgoSpeak/MiniROAD/criterions/__init__.py
vendored
Executable file
@@ -0,0 +1,2 @@
|
||||
from .loss import OadLoss
|
||||
from .loss_builder import build_criterion
|
||||
205
third_party/EgoSpeak/MiniROAD/criterions/loss.py
vendored
Executable file
205
third_party/EgoSpeak/MiniROAD/criterions/loss.py
vendored
Executable file
@@ -0,0 +1,205 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from criterions.loss_builder import CRITERIONS
|
||||
|
||||
|
||||
@CRITERIONS.register('NONUNIFORM')
|
||||
class OadLoss(nn.Module):
|
||||
|
||||
def __init__(self, cfg, reduction='mean'):
|
||||
super(OadLoss, self).__init__()
|
||||
self.reduction = reduction
|
||||
self.num_classes = cfg['num_classes']
|
||||
self.loss_offset = cfg['loss_offset'] if 'loss_offset' in cfg else 0
|
||||
assert self.loss_offset==0, f'loss_offset is not 0: {self.loss_offset}'
|
||||
self.loss = self.end_loss
|
||||
|
||||
def end_loss(self, out_dict, target):
|
||||
# logits: (B, seq, K) target: (B, seq, K)
|
||||
assert self.loss_offset < out_dict['logits'].shape[1], f'loss_offset is longer than the sequence loss_offset: {self.loss_offset}, logits.shape: {out_dict["logits"].shape}'
|
||||
logits = out_dict['logits']
|
||||
logits = logits[:,-1 - self.loss_offset,:].contiguous()
|
||||
target = target[:,-1 - self.loss_offset,:].contiguous()
|
||||
ce_loss = self.mlce_loss(logits, target)
|
||||
return ce_loss
|
||||
|
||||
def mlce_loss(self, logits, target):
|
||||
'''
|
||||
multi label cross entropy loss.
|
||||
logits: (B, K) target: (B, K)
|
||||
'''
|
||||
logsoftmax = nn.LogSoftmax(dim=-1).to(logits.device)
|
||||
output = torch.sum(-F.normalize(target) * logsoftmax(logits), dim=1) # B
|
||||
if self.reduction == 'mean':
|
||||
loss = torch.mean(output)
|
||||
elif self.reduction == 'sum':
|
||||
loss = torch.sum(output)
|
||||
return loss
|
||||
|
||||
def forward(self, out_dict, target):
|
||||
return self.loss(out_dict, target)
|
||||
|
||||
|
||||
@CRITERIONS.register('UNIFORM')
|
||||
class OadUniformLoss(nn.Module):
|
||||
|
||||
def __init__(self, cfg, reduction='mean'):
|
||||
super(OadUniformLoss, self).__init__()
|
||||
self.reduction = reduction
|
||||
self.num_classes = cfg['num_classes']
|
||||
# self.loss_offset = cfg['loss_offset'] if 'loss_offset' in cfg else 0
|
||||
self.loss = self.end_loss
|
||||
|
||||
def end_loss(self, out_dict, target):
|
||||
# logits: (B, seq, K) target: (B, seq, K)
|
||||
logits = out_dict['logits']
|
||||
# logits = logits[:,-1 - self.loss_offset,:].contiguous()
|
||||
# target = target[:,-1 - self.loss_offset,:].contiguous()
|
||||
ce_loss = self.mlce_loss(logits, target)
|
||||
return ce_loss
|
||||
|
||||
def mlce_loss(self, logits, target):
|
||||
'''
|
||||
multi label cross entropy loss.
|
||||
logits: (B, K) target: (B, K)
|
||||
'''
|
||||
logsoftmax = nn.LogSoftmax(dim=-1).to(logits.device)
|
||||
output = torch.sum(-target * logsoftmax(logits), dim=1) # B
|
||||
# output = torch.sum(-F.normalize(target) * logsoftmax(logits), dim=1) # B
|
||||
if self.reduction == 'mean':
|
||||
loss = torch.mean(output)
|
||||
elif self.reduction == 'sum':
|
||||
loss = torch.sum(output)
|
||||
return loss
|
||||
|
||||
def forward(self, out_dict, target):
|
||||
return self.loss(out_dict, target)
|
||||
|
||||
@CRITERIONS.register('OADTR')
|
||||
class OadTRLoss(nn.Module):
|
||||
|
||||
def __init__(self, cfg, reduction='mean'):
|
||||
super(OadTRLoss, self).__init__()
|
||||
self.reduction = reduction
|
||||
self.num_classes = cfg['num_classes']
|
||||
# self.loss_offset = cfg['loss_offset'] if 'loss_offset' in cfg else 0
|
||||
self.loss = self.end_loss
|
||||
|
||||
def end_loss(self, out_dict, target):
|
||||
# logits: (B, seq, K) target: (B, seq, K)
|
||||
assert self.loss_offset < out_dict['logits'].shape[1], f'loss_offset is longer than the sequence loss_offset: {self.loss_offset}, logits.shape: {out_dict["logits"].shape}'
|
||||
logits = out_dict['logits']
|
||||
# logits = logits[:,-1 - self.loss_offset,:].contiguous()
|
||||
# target = target[:,-1 - self.loss_offset,:].contiguous()
|
||||
ce_loss = self.mlce_loss(logits, target)
|
||||
return ce_loss
|
||||
|
||||
def mlce_loss(self, logits, target):
|
||||
'''
|
||||
multi label cross entropy loss.
|
||||
logits: (B, K) target: (B, K)
|
||||
'''
|
||||
logsoftmax = nn.LogSoftmax(dim=-1).to(logits.device)
|
||||
output = torch.sum(-F.normalize(target) * logsoftmax(logits), dim=1) # B
|
||||
if self.reduction == 'mean':
|
||||
loss = torch.mean(output)
|
||||
elif self.reduction == 'sum':
|
||||
loss = torch.sum(output)
|
||||
return loss
|
||||
|
||||
def forward(self, out_dict, enc_target, dec_target):
|
||||
return self.loss(out_dict, enc_target, dec_target)
|
||||
|
||||
@CRITERIONS.register('ANTICIPATION')
|
||||
class OadAntLoss(nn.Module):
|
||||
|
||||
# reduction was sum, not mean, what if we change to sum?
|
||||
def __init__(self, cfg, reduction='mean'):
|
||||
super(OadAntLoss, self).__init__()
|
||||
self.reduction = reduction
|
||||
self.loss = self.anticipation_loss
|
||||
self.num_classes = cfg['num_classes']
|
||||
|
||||
def anticipation_loss(self, out_dict, target, ant_target):
|
||||
# anticipation_logits: (B, seq, ANTICIPATION_LENGTH, num_class)
|
||||
# ant_target: (B, ANTICIPATION_LENGTH, num_class)
|
||||
anticipation_logits = out_dict['anticipation_logits']
|
||||
# [B, window_size, anticipation_length, num_class]
|
||||
pred_anticipation_logits = anticipation_logits[:,-1,:,:].contiguous().view(-1, self.num_classes)
|
||||
|
||||
anticipation_logit_targets = ant_target.view(-1, self.num_classes)
|
||||
ant_loss = self.mlce_loss(pred_anticipation_logits, anticipation_logit_targets)
|
||||
|
||||
return ant_loss
|
||||
|
||||
def ce_loss(self, out_dict, target):
|
||||
# logits: (B, seq, K) target: (B, seq, K)
|
||||
logits = out_dict['logits']
|
||||
logits = logits[:,-1,:].contiguous()
|
||||
target = target[:,-1,:].contiguous()
|
||||
ce_loss = self.mlce_loss(logits, target)
|
||||
return ce_loss
|
||||
|
||||
def mlce_loss(self, logits, target):
|
||||
'''
|
||||
multi label cross entropy loss.
|
||||
logits: (B, K) target: (B, K)
|
||||
'''
|
||||
logsoftmax = nn.LogSoftmax(dim=-1).to(logits.device)
|
||||
output = torch.sum(-F.normalize(target) * logsoftmax(logits), dim=1) # B
|
||||
if self.reduction == 'mean':
|
||||
loss = torch.mean(output)
|
||||
elif self.reduction == 'sum':
|
||||
loss = torch.sum(output)
|
||||
|
||||
return loss
|
||||
|
||||
def forward(self, out_dict, target, ant_target):
|
||||
return self.loss(out_dict, target, ant_target)
|
||||
|
||||
|
||||
|
||||
@CRITERIONS.register('UNIFORM_ANTICIPATION')
|
||||
class OadUniformAntLoss(nn.Module):
|
||||
|
||||
# reduction was sum, not mean, what if we change to sum?
|
||||
def __init__(self, cfg, reduction='mean'):
|
||||
super(OadUniformAntLoss, self).__init__()
|
||||
self.reduction = reduction
|
||||
self.loss = self.anticipation_loss
|
||||
self.num_classes = cfg['num_classes']
|
||||
|
||||
def anticipation_loss(self, out_dict, target, ant_target):
|
||||
anticipation_logits = out_dict['anticipation_logits']
|
||||
pred_anticipation_logits = anticipation_logits[:,-1,:,:].contiguous().view(-1, self.num_classes)
|
||||
anticipation_logit_targets = ant_target.view(-1, self.num_classes)
|
||||
ant_loss = self.mlce_loss(pred_anticipation_logits, anticipation_logit_targets)
|
||||
return ant_loss
|
||||
|
||||
def ce_loss(self, out_dict, target):
|
||||
# logits: (B, seq, K) target: (B, seq, K)
|
||||
logits = out_dict['logits']
|
||||
logits = logits[:,-1,:].contiguous()
|
||||
target = target[:,-1,:].contiguous()
|
||||
ce_loss = self.mlce_loss(logits, target)
|
||||
return ce_loss
|
||||
|
||||
def mlce_loss(self, logits, target):
|
||||
'''
|
||||
multi label cross entropy loss.
|
||||
logits: (B, K) target: (B, K)
|
||||
'''
|
||||
logsoftmax = nn.LogSoftmax(dim=-1).to(logits.device)
|
||||
output = torch.sum(-target * logsoftmax(logits), dim=1) # B
|
||||
|
||||
# output = torch.sum(-F.normalize(target) * logsoftmax(logits), dim=1) # B
|
||||
if self.reduction == 'mean':
|
||||
loss = torch.mean(output)
|
||||
elif self.reduction == 'sum':
|
||||
loss = torch.sum(output)
|
||||
|
||||
return loss
|
||||
|
||||
def forward(self, out_dict, target, ant_target):
|
||||
return self.loss(out_dict, target, ant_target)
|
||||
11
third_party/EgoSpeak/MiniROAD/criterions/loss_builder.py
vendored
Executable file
11
third_party/EgoSpeak/MiniROAD/criterions/loss_builder.py
vendored
Executable file
@@ -0,0 +1,11 @@
|
||||
__all__ = [
|
||||
'build_criterion'
|
||||
]
|
||||
|
||||
from utils import Registry
|
||||
|
||||
CRITERIONS = Registry()
|
||||
|
||||
def build_criterion(cfg, device=None):
|
||||
criterion = CRITERIONS[cfg["loss"]](cfg)
|
||||
return criterion.to(device)
|
||||
426
third_party/EgoSpeak/MiniROAD/data_info/YTConv_info.json
vendored
Normal file
426
third_party/EgoSpeak/MiniROAD/data_info/YTConv_info.json
vendored
Normal file
@@ -0,0 +1,426 @@
|
||||
{
|
||||
"shard_0": {
|
||||
"class_index": [
|
||||
"background",
|
||||
"camera wearer start to speak"
|
||||
],
|
||||
"train_session_set": [
|
||||
"m5Aoywf51L4",
|
||||
"p99cIB7nHjw",
|
||||
"zGgMzBfPfaI",
|
||||
"O0eCtKU3FxY",
|
||||
"goZQ6iOIFYo",
|
||||
"9Tb-lSyhgoE",
|
||||
"GBxQtGUjAMA",
|
||||
"VXgt7keOC8I",
|
||||
"hZ2tin2J0P4",
|
||||
"Ur-P18sm4tI",
|
||||
"WfYmsNBQwg8",
|
||||
"AvMNaDDfrN0",
|
||||
"fW7pCN3wwdo",
|
||||
"xG6HBdTtS4s",
|
||||
"HcDlqUeDD9Q",
|
||||
"1DE3mKncHUA",
|
||||
"oEMX_quSMZY",
|
||||
"jmS3nWbGV5Y",
|
||||
"qUJRjPRH-u4",
|
||||
"aHsUgWfUcss",
|
||||
"ZfdQOmJkaK0",
|
||||
"4f43nYrfTnI",
|
||||
"qE1FnzkNjrE",
|
||||
"4NsmKhs9flk",
|
||||
"3wDSJ9ABzmo",
|
||||
"sY6_8QwRZsw",
|
||||
"hByfAzEoeKA",
|
||||
"vDEsV0AhcjM",
|
||||
"XGNhX3Nu1RE",
|
||||
"MkVT8K0V57A",
|
||||
"yR8y-wN8Clk",
|
||||
"P1dTOlO2sl8",
|
||||
"6UGC_wxTeB0",
|
||||
"d4Xud2Gqoxk",
|
||||
"IJ1aqePculQ",
|
||||
"o8Oxf5QBitQ",
|
||||
"XVci_V4qxmc",
|
||||
"ofDzhOS1Pv0",
|
||||
"mygTXBuMRF0",
|
||||
"iq6q82Jo8ZQ",
|
||||
"Kzc_ekBJ4QE",
|
||||
"WEeJ9B1i8HI",
|
||||
"rciqpxdCc9Y",
|
||||
"Rh7d2D7JVLI",
|
||||
"Qkx4hSlc8jk",
|
||||
"ixi7sVLFrWM",
|
||||
"n96-scSyoOQ",
|
||||
"YZ5xYioIQdA",
|
||||
"X76QxuWm1oA",
|
||||
"2I7d_LfK75Y",
|
||||
"AeUWBaYFMyo",
|
||||
"lsqXFgymLlI",
|
||||
"Dx_0jYMWN5o",
|
||||
"0raJ1-1BJ7s",
|
||||
"EGX8am5hWh0",
|
||||
"KjtZ_CLY1hQ",
|
||||
"R4FQI2BI3hw",
|
||||
"fAoqqtsNjbQ",
|
||||
"l6RwyA6jUdM",
|
||||
"GJW6dlagRnU",
|
||||
"j_FGvmbp6aM",
|
||||
"V2O15h1VrH8",
|
||||
"P3c70VbDP3E",
|
||||
"epAwpBdUiYU",
|
||||
"6dpv_JeyL8w",
|
||||
"dUKYxsvnuBM",
|
||||
"L2_zn7h8sOQ",
|
||||
"m09qdCkAhmY",
|
||||
"7BB71tIae7A",
|
||||
"6go4oViEkdc",
|
||||
"MYimxshUK1c",
|
||||
"v4eZXWcIRQU",
|
||||
"vGI_w1Ml6S0",
|
||||
"BeFndnkT00c",
|
||||
"9JHRbL1jmq4",
|
||||
"0rBaR5dd-Fs",
|
||||
"EvhkIS_-K-w",
|
||||
"pYVbLx-h5HI",
|
||||
"fVciOXYDlqg",
|
||||
"zWFH4N-CmDE",
|
||||
"Aw4cauzfX5E",
|
||||
"lxVpcQHtcWs",
|
||||
"32dkF-aXgug",
|
||||
"_USorgWOsUk",
|
||||
"1yVMU93nmv8",
|
||||
"aYkF4Ybk-l8",
|
||||
"Vwt18p0BqzM",
|
||||
"oOQM5Pvjut4",
|
||||
"_WWSMNtNYx0",
|
||||
"zzlhwg0IJ5Y",
|
||||
"lJhIffkUhr8",
|
||||
"PbS9nSH3mqM",
|
||||
"8EGVgiFpwhg",
|
||||
"5LhXpv5IFO4",
|
||||
"ibD4APVzWgg",
|
||||
"2nhcSQmopO4",
|
||||
"gri79OfCRvo",
|
||||
"lOwQN3QcRzE",
|
||||
"wQyluRNtqFo",
|
||||
"IZyx8zeiKB0",
|
||||
"mZyNfKyzDHU",
|
||||
"A9eGJWiuIKs",
|
||||
"d37DZogot1w",
|
||||
"7ZUSVabB6N4",
|
||||
"m62GQXq-358",
|
||||
"ArwPVsCZo2c",
|
||||
"QK-BFkpO8s4",
|
||||
"PgfX9bvmEhs",
|
||||
"l1jTUhwWJYA",
|
||||
"S1PGHO4evaI",
|
||||
"M6uPJa5B8Ns",
|
||||
"XXzz19J-1w8",
|
||||
"U4uV1xmUaB0",
|
||||
"skQ0OBpkKqY",
|
||||
"mfW57TDpMtc",
|
||||
"NAwKA75cDwE",
|
||||
"eIEBI2_Y14Q",
|
||||
"eWQGYpSoiYU",
|
||||
"3DPIUWhYtn8",
|
||||
"_elvaRJrhE0",
|
||||
"PAA8IFhWJzM",
|
||||
"1XDKqhq6bb0",
|
||||
"ngi9gy3DzRY",
|
||||
"Y0yONfuimeE",
|
||||
"g9_cAgHa_4Q",
|
||||
"BIG7jEw-j4w",
|
||||
"TFdom0lkd-0",
|
||||
"2zV2N8_DwWg",
|
||||
"kTz7XB_jpB4",
|
||||
"GuoglIuhNjY",
|
||||
"L98U3mO38gQ",
|
||||
"UfWDoRAzpC8",
|
||||
"W6TakaXCIIQ",
|
||||
"i6j1I6pha-M",
|
||||
"issHFC8Gbio",
|
||||
"4EzGoRkd6SI",
|
||||
"QoS0GgF4sI4",
|
||||
"A8hJg3D_TGY",
|
||||
"A3N10suCRCw",
|
||||
"6962tnWRsNY",
|
||||
"lS4p8osXSJs",
|
||||
"wbZ7ZhcZd6o",
|
||||
"fsIvQvIlKwc",
|
||||
"4jQSOSi2DA8",
|
||||
"Vpt6tZBvXQU",
|
||||
"kxOG3kpLmYw",
|
||||
"Mzjlz0nq9AA",
|
||||
"80cuXt2VSko",
|
||||
"VvswhogSiY0",
|
||||
"5Q6ky5VrOIA",
|
||||
"I_XtcUpNRF8",
|
||||
"QmtmTGycK9A",
|
||||
"qp3x8iy7vkA",
|
||||
"xGU8QDtPrQI",
|
||||
"uWWUfqth_BA",
|
||||
"WbM7w8r4VGQ",
|
||||
"bt6Ed1qjulw",
|
||||
"7vymT27scoY",
|
||||
"b9AdV6oVDVg",
|
||||
"nf2zqYUAOwg",
|
||||
"A-LNJc-1Jc0",
|
||||
"yCI-hpPiDVc",
|
||||
"Uu5DME4FqIM",
|
||||
"symjuH9iOTc",
|
||||
"mPf2MTnTBc0",
|
||||
"NDTlQ1KGvH8",
|
||||
"g9vuDzFQSRU",
|
||||
"B6iqnvleve0",
|
||||
"9bbszAD7JL0",
|
||||
"RIyMAuz7Ckg",
|
||||
"mlrQHtBRMhc",
|
||||
"3gaidlW7vig",
|
||||
"DGdV97q9ndk",
|
||||
"qthjlg0lXFg",
|
||||
"gWng5YLPOow",
|
||||
"iaYG-KL9XRM",
|
||||
"385414AVZcA",
|
||||
"7xhBnWGpI7c",
|
||||
"Nh0cmSQUs-E",
|
||||
"Rvtj0QHPulA",
|
||||
"rAtLksYQukw",
|
||||
"S9rB__zbbe0",
|
||||
"SNpMVsOPfHM",
|
||||
"zhF43F4kO2w",
|
||||
"HOUkcInyxN0",
|
||||
"x8ymkfr2Z1o",
|
||||
"Ix6Khtq6O-k",
|
||||
"VHs_JyPhmtE",
|
||||
"zJX3byjYSMc",
|
||||
"zdqX09zZNf8",
|
||||
"uPwlmwNQq8Y",
|
||||
"e4xNTXrk6wE",
|
||||
"P2PNUWN4GMs",
|
||||
"_WVHCYjHBVY",
|
||||
"5Jtv1GpPvHM",
|
||||
"AIZYucKOzhw",
|
||||
"vbsR_VDZXh0",
|
||||
"9Ew0lDHB0kM",
|
||||
"ejpkPVZHuMM",
|
||||
"47HlvCfubE4",
|
||||
"HzPkQSIG2m0",
|
||||
"e7mTRnNnOXg",
|
||||
"eZHx_biEECg",
|
||||
"FvP6WFKNP4Y",
|
||||
"g_TJ5XFEVfY",
|
||||
"8Utg2VEWsJg",
|
||||
"VMdBjL-tpoo",
|
||||
"zihP2Eqeqxk",
|
||||
"ILVqgOlntLM",
|
||||
"Z1BWQI8KPCg",
|
||||
"j8TRVp-7l3Q",
|
||||
"G5gJBZ-3OlY",
|
||||
"o1JUlGSirSE",
|
||||
"OsPdhUQ0snE",
|
||||
"-5uIaW2gKWQ",
|
||||
"03kiFNh-9ME",
|
||||
"w5W6mgDaJJA",
|
||||
"sYsgd_8AFoo",
|
||||
"vLg8kVUTmWU",
|
||||
"OK0J0xsuSWQ",
|
||||
"d284rjoU4Cs",
|
||||
"5lE6V4jrUl4",
|
||||
"bS1OpC9xu48",
|
||||
"ZUxpqU_J_n0",
|
||||
"AUliC0c_YIQ",
|
||||
"nJ58-d_viOo",
|
||||
"uIy3b2kzlrM",
|
||||
"vOTLfb0qjfY",
|
||||
"37_7tPZjC34",
|
||||
"MUwCdP5qceI",
|
||||
"lYvNi_ewd8U",
|
||||
"PDG7SzDBTC4",
|
||||
"QHQzoHolWvI",
|
||||
"YmvOEj00LB8",
|
||||
"fLhkXVx7Yw0",
|
||||
"Uj5nhugVgwo",
|
||||
"aHbrDu2o0WY",
|
||||
"yKW6vVgEPCM",
|
||||
"Q0WUgyqQ-4I",
|
||||
"xw3KOPJ9a3E",
|
||||
"5mrTo7c3bCM",
|
||||
"x0ft032W--4",
|
||||
"UdQxEbrIocc",
|
||||
"EHZHAFyv_Ks",
|
||||
"7DojKgPRLVk",
|
||||
"-1CahRD6lJ0",
|
||||
"f3TuYwl7AZE",
|
||||
"TgGTovCcS9Y",
|
||||
"AwyrJGeIN_g",
|
||||
"RbPJD4Wh-l8",
|
||||
"58bFdwYkDzU",
|
||||
"6gIEKGxL7zo",
|
||||
"BeuzJwTuChQ",
|
||||
"YebZyAzLZuc",
|
||||
"9T74b8EQBVY",
|
||||
"TqSVFjw1uNM",
|
||||
"WUYhCmfE1a0",
|
||||
"vRCVUQFUio4",
|
||||
"w2t9JdMNMgQ",
|
||||
"nn6-x9xXMeE",
|
||||
"FDFVPzLlzFs",
|
||||
"qFhvi4bH29E",
|
||||
"qdpsMXLsUZM",
|
||||
"JREk1QeSc9A",
|
||||
"jOVEs-4r1jc",
|
||||
"v4VZI9Ouzy0",
|
||||
"DMg5UUvf-C4",
|
||||
"26J5zRzQkOg",
|
||||
"YkiJ4iJfJbw",
|
||||
"_d8j8nfTEIk",
|
||||
"wMNb3hQXEUE",
|
||||
"1KI8ex22blk",
|
||||
"eCp2Jp7MtKg",
|
||||
"HNlpHBb5dC4",
|
||||
"C6_pDRdIo1k",
|
||||
"ZRWJJHrgP54",
|
||||
"L7jfx4veKIA",
|
||||
"p1a5_qltICM",
|
||||
"nZ0svM8DoY0",
|
||||
"miEzjCJVaZk",
|
||||
"TpC2tDCxABw",
|
||||
"SZmqoqCX__U",
|
||||
"dZ0bYteaI2Q",
|
||||
"GW-cYEzShmY",
|
||||
"iUYuPhutKVw",
|
||||
"97VSm6T2zMw",
|
||||
"jPiQyi5jr1M",
|
||||
"lrUi4nsPZ9M",
|
||||
"rXLeXke-k_w",
|
||||
"mEO4izUqZ7E",
|
||||
"4qI1LjMXfh0",
|
||||
"mVt6myGMa-0",
|
||||
"VJrnhh6FqEg",
|
||||
"vwCeBWbZvrE",
|
||||
"c08fl8-xmVE",
|
||||
"syZ3bvXIrpA",
|
||||
"i3ysNaulop4",
|
||||
"__t3rU0QjsE",
|
||||
"jT32uQQVGvU",
|
||||
"Fzvmh-R2EWk",
|
||||
"QApJSKrLg6E",
|
||||
"L5y3j8SdGSU",
|
||||
"3t8JGblR__0",
|
||||
"e1F90W40t6g",
|
||||
"j6L8UIa5z28",
|
||||
"Zmg2_C1FuVI",
|
||||
"XIXKJXsiBFc",
|
||||
"NVHB1VdWFo4",
|
||||
"-x7-8f7bsl4",
|
||||
"2FSxSiowkGI",
|
||||
"jgFIBIeFve0",
|
||||
"1BLuXK1M5CE",
|
||||
"zmpx6LUT2ow",
|
||||
"z-QUzJTsFro",
|
||||
"0sS4v13DAK8",
|
||||
"gMIBQ0mbRvM",
|
||||
"9kcaEXlTB5s",
|
||||
"ZYArPGMLS4k",
|
||||
"tejW9GMaucA",
|
||||
"z9r4bmXCWiI",
|
||||
"4Zpsb46iaLc",
|
||||
"52Odc8UbvC8",
|
||||
"x-B6-iYDHPs",
|
||||
"8GinSDc0Is8",
|
||||
"5lwbp5d_w0Y",
|
||||
"I4LpTQLzlHE",
|
||||
"lMl6-XH7ozY",
|
||||
"tOubscNbY1Q",
|
||||
"7gb7AH-oWcQ",
|
||||
"mDerqt9Ib-I",
|
||||
"gKV3FNd9WHE",
|
||||
"kH5L1E_r-3U",
|
||||
"sq5TRFxVp0w",
|
||||
"SrBhj7Xusk8",
|
||||
"RxlxEapsz2E",
|
||||
"aCD3_TziiKo",
|
||||
"_HPVZOkxdkk",
|
||||
"pNCjgVQ53Y8",
|
||||
"iPdT5GUoGvA",
|
||||
"cIrlet1oRw4",
|
||||
"1dyWUMBjMwQ",
|
||||
"XZQB3cq18yc",
|
||||
"i75Mn_AeFBY",
|
||||
"_0VwR9WPS-k",
|
||||
"q1JDjgnRdGY",
|
||||
"1JNEpfvRq0Y",
|
||||
"2FGj6HciQZA",
|
||||
"eLi59TtNwHY",
|
||||
"Z3W8OST7tmA",
|
||||
"F2dNYov8-xw",
|
||||
"kPfyJu_CrSA",
|
||||
"7I4Y2HtAwlY",
|
||||
"us-andJDfWw",
|
||||
"Q0Bzn1teWfo",
|
||||
"pADxBfOrYeQ",
|
||||
"KnA1tIR0qCE",
|
||||
"lBNPfBcb6pw",
|
||||
"zt9MIiBfxF8",
|
||||
"FwX12hPqCws",
|
||||
"6e2LXGrze2M",
|
||||
"0dvclhcFjSo",
|
||||
"UTkuDYOIxKo",
|
||||
"4n044cOTiNg",
|
||||
"VxksG15OgRY",
|
||||
"4193MkWdJcQ",
|
||||
"2bDX4Uzy8Wc",
|
||||
"SkyBNX2auEc",
|
||||
"JrSLspuJ2gk",
|
||||
"GZO7V-CWSJ8",
|
||||
"TYMIy4qlRzc",
|
||||
"jK7q_Li7LU0",
|
||||
"UiTIXDQEkPM"
|
||||
],
|
||||
"test_session_set": [
|
||||
"DTGtC7FC4oI",
|
||||
"CIWvk3gJ_7E",
|
||||
"A1fZ6rlOwxk",
|
||||
"cB8WCrmJEEA",
|
||||
"gGQ_2Ank9-U",
|
||||
"b91abcyMeNY",
|
||||
"9MuaFeyczM0",
|
||||
"rwZxhdK_UVU",
|
||||
"hd_WbrqYnTE",
|
||||
"4BWcqA9cFFk",
|
||||
"MuOkgCkob6E",
|
||||
"OdhFWA3Fgb0",
|
||||
"Pe3ZabrUiUk",
|
||||
"U0c_uv8DU08",
|
||||
"kTqt_3YJBbs",
|
||||
"eWXVVMi4bf8",
|
||||
"RCD-vXumMIs",
|
||||
"DuwiezJCZ3I",
|
||||
"BNgEqjAiKU4",
|
||||
"Cbn0t7eqaEs",
|
||||
"D3M9Q0WwuFY",
|
||||
"oO-Bv4iRZKI",
|
||||
"bBvqtbZdSZQ",
|
||||
"-mnoItCi32Y",
|
||||
"yQPxKfz-bwI",
|
||||
"ITVZ4cx5big",
|
||||
"uY5_e0L7mjw",
|
||||
"J_Pl9ftnNXE",
|
||||
"Jt_pb40MRy4",
|
||||
"E3hAj2niRNo",
|
||||
"kl2xVSxVECQ",
|
||||
"gzebuFs90Xg",
|
||||
"6DQSIzJ9J-g",
|
||||
"pJJlnryTYWg",
|
||||
"X87qsfVLFoc",
|
||||
"W7BIPWU1WHc",
|
||||
"Oj506EVOiQc",
|
||||
"DPEvWvcstxA",
|
||||
"Pk16UT5-zeM",
|
||||
"XMK1KEk5OdU",
|
||||
"PpNm5llsEW8",
|
||||
"sAp3rvablGk"
|
||||
]
|
||||
}
|
||||
}
|
||||
1898
third_party/EgoSpeak/MiniROAD/data_info/data_info.json
vendored
Normal file
1898
third_party/EgoSpeak/MiniROAD/data_info/data_info.json
vendored
Normal file
File diff suppressed because it is too large
Load Diff
2
third_party/EgoSpeak/MiniROAD/datasets/__init__.py
vendored
Executable file
2
third_party/EgoSpeak/MiniROAD/datasets/__init__.py
vendored
Executable file
@@ -0,0 +1,2 @@
|
||||
from .dataset_builder import build_dataset, build_data_loader
|
||||
from .dataset import THUMOSDataset, FINEACTIONDataset
|
||||
314
third_party/EgoSpeak/MiniROAD/datasets/dataset.py
vendored
Executable file
314
third_party/EgoSpeak/MiniROAD/datasets/dataset.py
vendored
Executable file
@@ -0,0 +1,314 @@
|
||||
import gc
|
||||
import json
|
||||
import os.path as osp
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.utils.data as data
|
||||
from configs.feature_sizes import FEATURE_SIZES
|
||||
from datasets.dataset_builder import DATA_LAYERS
|
||||
from utils.util import parse_flow_type, parse_rgb_type
|
||||
|
||||
|
||||
@DATA_LAYERS.register("THUMOS")
|
||||
@DATA_LAYERS.register("TVSERIES")
|
||||
@DATA_LAYERS.register("EasyCom")
|
||||
@DATA_LAYERS.register("EGO4D")
|
||||
@DATA_LAYERS.register("IEMOCAP")
|
||||
@DATA_LAYERS.register("IEMOCAP_DEBUG")
|
||||
@DATA_LAYERS.register("YTConv")
|
||||
class THUMOSDataset(data.Dataset):
|
||||
|
||||
def __init__(self, cfg, mode='train'):
|
||||
self.root_path = cfg['root_path']
|
||||
self.mode = mode
|
||||
self.training = mode == 'train'
|
||||
self.window_size = cfg['window_size']
|
||||
self.enc_steps = cfg['window_size'] if 'window_size' in cfg else None
|
||||
self.dec_steps = cfg['query_num'] if 'query_num' in cfg else None
|
||||
|
||||
self.stride = cfg['stride']
|
||||
data_name = cfg['data_name']
|
||||
self.model = cfg['model']
|
||||
self.vids = json.load(open(cfg['video_list_path']))[data_name][mode + '_session_set']
|
||||
self.num_classes = cfg['num_classes']
|
||||
self.inputs = []
|
||||
|
||||
self._load_features(cfg)
|
||||
self._init_features()
|
||||
|
||||
def _load_features(self, cfg):
|
||||
self.annotation_type = cfg['annotation_type']
|
||||
self.rgb_type = cfg['rgb_type']
|
||||
self.flow_type = cfg['flow_type']
|
||||
self.target_all = {}
|
||||
self.rgb_inputs = {}
|
||||
self.flow_inputs = {}
|
||||
dummy_target = np.zeros((self.window_size-1, self.num_classes))
|
||||
|
||||
rgb_type = parse_rgb_type(cfg['rgb_type'])
|
||||
flow_type = parse_flow_type(cfg['flow_type'])
|
||||
|
||||
dummy_rgb = np.zeros((self.window_size-1, FEATURE_SIZES[rgb_type]))
|
||||
dummy_flow = np.zeros((self.window_size-1, FEATURE_SIZES[flow_type]))
|
||||
for vid in self.vids:
|
||||
target = np.load(osp.join(self.root_path, self.annotation_type, vid + '.npy'))
|
||||
rgb = np.load(osp.join(self.root_path, self.rgb_type, vid + '.npy'))
|
||||
flow = np.load(osp.join(self.root_path, self.flow_type, vid + '.npy'))
|
||||
|
||||
if self.training:
|
||||
self.target_all[vid] = np.concatenate((dummy_target, target), axis=0)
|
||||
self.rgb_inputs[vid] = np.concatenate((dummy_rgb, rgb), axis=0)
|
||||
self.flow_inputs[vid] = np.concatenate((dummy_flow, flow), axis=0)
|
||||
else:
|
||||
self.target_all[vid] = target
|
||||
self.rgb_inputs[vid] = rgb
|
||||
self.flow_inputs[vid] = flow
|
||||
|
||||
def _init_features(self):
|
||||
del self.inputs
|
||||
gc.collect()
|
||||
self.inputs = []
|
||||
if self.model == 'MiniROAD':
|
||||
for vid in self.vids:
|
||||
target = self.target_all[vid]
|
||||
|
||||
|
||||
if self.training:
|
||||
seed = np.random.randint(self.stride)
|
||||
for start, end in zip(range(seed, target.shape[0], self.stride),
|
||||
range(seed + self.window_size, target.shape[0]+1, self.stride)):
|
||||
self.inputs.append([
|
||||
vid, start, end, target[start:end]
|
||||
])
|
||||
|
||||
else:
|
||||
start = 0
|
||||
end = target.shape[0]
|
||||
self.inputs.append([
|
||||
vid, start, end, target[start:end]
|
||||
])
|
||||
|
||||
elif self.model == 'OADTR':
|
||||
for vid in self.vids:
|
||||
target = self.target_all[vid]
|
||||
|
||||
|
||||
seed = np.random.randint(self.stride) if self.training else 0
|
||||
|
||||
for start, end in zip(
|
||||
range(seed, target.shape[0], 1),
|
||||
range(seed + self.window_size, target.shape[0]+1, 1)
|
||||
):
|
||||
enc_target = target[start:end]
|
||||
dec_target = target[end:end + self.dec_steps]
|
||||
distance_target, class_h_target = self.get_distance_target(target[start:end])
|
||||
|
||||
self.inputs.append([
|
||||
vid, start, end, enc_target, distance_target, class_h_target, dec_target
|
||||
])
|
||||
|
||||
else:
|
||||
for vid in self.vids:
|
||||
target = self.target_all[vid]
|
||||
seed = np.random.randint(self.stride) if self.training else 0
|
||||
for start, end in zip(
|
||||
range(seed, target.shape[0], 1),
|
||||
range(seed + self.window_size, target.shape[0]+1, 1)
|
||||
):
|
||||
self.inputs.append([
|
||||
vid, start, end, target[start:end]
|
||||
])
|
||||
|
||||
def get_distance_target(self, target_vector):
|
||||
target_matrix = np.zeros(self.enc_steps - 1)
|
||||
target_argmax = target_vector[self.enc_steps-1].argmax()
|
||||
for i in range(self.enc_steps - 1):
|
||||
if target_vector[i].argmax() == target_argmax:
|
||||
target_matrix[i] = 1.
|
||||
return target_matrix, target_vector[self.enc_steps-1]
|
||||
|
||||
def __getitem__(self, index):
|
||||
if self.model == 'MiniROAD' or self.model == 'Transformer':
|
||||
vid, start, end, target = self.inputs[index]
|
||||
rgb_input = self.rgb_inputs[vid][start:end]
|
||||
flow_input = self.flow_inputs[vid][start:end]
|
||||
rgb_input = torch.tensor(rgb_input.astype(np.float32))
|
||||
flow_input = torch.tensor(flow_input.astype(np.float32))
|
||||
target = torch.tensor(target.astype(np.float32))
|
||||
return vid, rgb_input, flow_input, target
|
||||
|
||||
elif self.model == 'OADTR':
|
||||
vid, start, end, enc_target, distance_target, class_h_target, dec_target = self.inputs[index]
|
||||
rgb_input = self.rgb_inputs[vid][start:end]
|
||||
flow_input = self.flow_inputs[vid][start:end]
|
||||
rgb_input = torch.tensor(rgb_input.astype(np.float32))
|
||||
flow_input = torch.tensor(flow_input.astype(np.float32))
|
||||
enc_target = torch.tensor(enc_target.astype(np.float32))
|
||||
distance_target = torch.tensor(distance_target.astype(np.float32))
|
||||
class_h_target = torch.tensor(class_h_target.astype(np.float32))
|
||||
dec_target = torch.tensor(dec_target.astype(np.float32))
|
||||
return vid, rgb_input, flow_input, enc_target, distance_target, class_h_target, dec_target
|
||||
|
||||
def __len__(self):
|
||||
return len(self.inputs)
|
||||
|
||||
|
||||
@DATA_LAYERS.register("THUMOS_ANTICIPATION")
|
||||
@DATA_LAYERS.register("TVSERIES_ANTICIPATION")
|
||||
@DATA_LAYERS.register("IEMOCAP_ANTICIPATION")
|
||||
@DATA_LAYERS.register("EasyCom_ANTICIPATION")
|
||||
@DATA_LAYERS.register("YTConv_ANTICIPATION")
|
||||
@DATA_LAYERS.register("EGO4D_ANTICIPATION")
|
||||
class THUMOSDataset(data.Dataset):
|
||||
|
||||
def __init__(self, cfg, mode='train'):
|
||||
self.cfg = cfg
|
||||
self.root_path = cfg['root_path']
|
||||
self.mode = mode
|
||||
self.training = mode == 'train'
|
||||
self.window_size = cfg['window_size']
|
||||
self.stride = cfg['stride']
|
||||
self.anticipation_length = cfg['anticipation_length']
|
||||
data_name = cfg["data_name"].split('_')[0]
|
||||
if 'data_info_name' in cfg.keys() and cfg['data_info_name'] is not None:
|
||||
data_name = cfg['data_info_name']
|
||||
|
||||
self.vids = json.load(open(cfg['video_list_path']))[data_name][mode + '_session_set']
|
||||
print(f'data_name: {data_name}')
|
||||
print(f'{mode} session set length: {len(self.vids)}')
|
||||
self.num_classes = cfg['num_classes']
|
||||
self.inputs = []
|
||||
|
||||
self._load_features(cfg)
|
||||
self._init_features()
|
||||
|
||||
def _load_features(self, cfg):
|
||||
self.annotation_type = cfg['annotation_type']
|
||||
self.rgb_type = cfg['rgb_type']
|
||||
self.flow_type = cfg['flow_type']
|
||||
|
||||
self.target_all = {}
|
||||
self.rgb_inputs = {}
|
||||
self.flow_inputs = {}
|
||||
|
||||
dummy_target = np.zeros((self.window_size-1, self.num_classes))
|
||||
dummy_rgb = np.zeros((self.window_size-1, FEATURE_SIZES[cfg['rgb_type']]))
|
||||
dummy_flow = np.zeros((self.window_size-1, FEATURE_SIZES[cfg['flow_type']]))
|
||||
|
||||
for vid in self.vids:
|
||||
target = np.load(osp.join(self.root_path, self.annotation_type, vid + '.npy'))
|
||||
rgb = np.load(osp.join(self.root_path, self.rgb_type, vid + '.npy'))
|
||||
flow = np.load(osp.join(self.root_path, self.flow_type, vid + '.npy'))
|
||||
|
||||
if self.training:
|
||||
self.target_all[vid] = np.concatenate((dummy_target, target), axis=0)
|
||||
self.rgb_inputs[vid] = np.concatenate((dummy_rgb, rgb), axis=0)
|
||||
self.flow_inputs[vid] = np.concatenate((dummy_flow, flow), axis=0)
|
||||
else:
|
||||
self.target_all[vid] = target
|
||||
self.rgb_inputs[vid] = rgb
|
||||
self.flow_inputs[vid] = flow
|
||||
|
||||
def _init_features(self):
|
||||
del self.inputs
|
||||
gc.collect()
|
||||
self.inputs = []
|
||||
|
||||
for vid in self.vids:
|
||||
target = self.target_all[vid]
|
||||
if self.training:
|
||||
seed = np.random.randint(self.stride)
|
||||
for start, end in zip(range(seed, target.shape[0], self.stride),
|
||||
range(seed + self.window_size, target.shape[0]-self.anticipation_length, self.stride)):
|
||||
self.inputs.append([
|
||||
vid, start, end, target[start:end], target[end:end+self.anticipation_length]
|
||||
])
|
||||
else:
|
||||
start = 0
|
||||
end = target.shape[0] - self.anticipation_length
|
||||
ant_target = []
|
||||
|
||||
for s in range(1, target.shape[0] - self.anticipation_length + 1):
|
||||
ant_target.append(target[s:s + self.anticipation_length])
|
||||
|
||||
self.inputs.append([
|
||||
vid, start, end, target[start:end], np.array(ant_target)
|
||||
])
|
||||
|
||||
|
||||
|
||||
def __getitem__(self, index):
|
||||
vid, start, end, target, ant_target = self.inputs[index]
|
||||
|
||||
|
||||
rgb_input = self.rgb_inputs[vid][start:end]
|
||||
flow_input = self.flow_inputs[vid][start:end]
|
||||
|
||||
rgb_input = torch.tensor(rgb_input.astype(np.float32))
|
||||
flow_input = torch.tensor(flow_input.astype(np.float32))
|
||||
|
||||
target = torch.tensor(target.astype(np.float32))
|
||||
ant_target = torch.tensor(ant_target.astype(np.float32))
|
||||
|
||||
return vid, rgb_input, flow_input, target, ant_target
|
||||
|
||||
def __len__(self):
|
||||
return len(self.inputs)
|
||||
|
||||
|
||||
@DATA_LAYERS.register("FINEACTION")
|
||||
class FINEACTIONDataset(data.Dataset):
|
||||
|
||||
def __init__(self, cfg, mode='train'):
|
||||
self.root_path = cfg['root_path']
|
||||
self.mode = mode
|
||||
self.training = mode == 'train'
|
||||
self.window_size = cfg['window_size']
|
||||
self.stride = cfg['stride']
|
||||
data_name = cfg['data_name']
|
||||
self.vids = json.load(open(cfg['video_list_path']))[data_name][mode + '_session_set']
|
||||
self.num_classes = cfg['num_classes']
|
||||
self.inputs = []
|
||||
self._load_features(cfg)
|
||||
self._init_features()
|
||||
|
||||
def _load_features(self, cfg):
|
||||
self.annotation_type = cfg['annotation_type']
|
||||
self.rgb_type = cfg['rgb_type']
|
||||
self.flow_type = cfg['flow_type']
|
||||
|
||||
def _init_features(self, seed=0):
|
||||
|
||||
del self.inputs
|
||||
gc.collect()
|
||||
self.inputs = []
|
||||
for vid in self.vids:
|
||||
target = np.load(osp.join(self.root_path, self.annotation_type, vid + '.npy'))
|
||||
if self.training:
|
||||
seed = np.random.randint(self.stride)
|
||||
for start, end in zip(range(seed, target.shape[0], self.stride),
|
||||
range(seed + self.window_size, target.shape[0]+1, self.stride)):
|
||||
self.inputs.append([
|
||||
vid, start, end
|
||||
])
|
||||
else:
|
||||
start = 0
|
||||
end = target.shape[0]
|
||||
self.inputs.append([
|
||||
vid, start, end
|
||||
])
|
||||
|
||||
def __getitem__(self, index):
|
||||
vid, start, end = self.inputs[index]
|
||||
rgb_input = np.load(osp.join(self.root_path, self.rgb_type, vid + '.npy'), mmap_mode='r')[start:end]
|
||||
flow_input = np.load(osp.join(self.root_path, self.flow_type, vid + '.npy'), mmap_mode='r')[start:end]
|
||||
target = np.load(osp.join(self.root_path, self.annotation_type, vid + '.npy'), mmap_mode='r')[start:end]
|
||||
rgb_input = torch.tensor(rgb_input.astype(np.float32))
|
||||
flow_input = torch.tensor(flow_input.astype(np.float32))
|
||||
target = torch.tensor(target.astype(np.float32))
|
||||
return rgb_input, flow_input, target
|
||||
|
||||
def __len__(self):
|
||||
return len(self.inputs)
|
||||
32
third_party/EgoSpeak/MiniROAD/datasets/dataset_builder.py
vendored
Executable file
32
third_party/EgoSpeak/MiniROAD/datasets/dataset_builder.py
vendored
Executable file
@@ -0,0 +1,32 @@
|
||||
__all__ = [
|
||||
'build_dataset',
|
||||
'build_data_loader',
|
||||
]
|
||||
|
||||
import torch.utils.data as data
|
||||
from utils import Registry
|
||||
|
||||
DATA_LAYERS = Registry()
|
||||
|
||||
def build_dataset(cfg):
|
||||
data_layer = DATA_LAYERS[f'{cfg["data_name"]}']
|
||||
return data_layer
|
||||
|
||||
def build_data_loader(cfg, mode):
|
||||
data_layer = build_dataset(cfg)
|
||||
data_loader = data.DataLoader(
|
||||
dataset=data_layer(cfg, mode),
|
||||
batch_size=cfg["batch_size"] if mode == 'train' else cfg["test_batch_size"],
|
||||
shuffle=True if mode == 'train' else False,
|
||||
num_workers=cfg["num_workers"],
|
||||
pin_memory=False,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
return data_loader
|
||||
294
third_party/EgoSpeak/MiniROAD/main.py
vendored
Executable file
294
third_party/EgoSpeak/MiniROAD/main.py
vendored
Executable file
@@ -0,0 +1,294 @@
|
||||
import argparse
|
||||
import os
|
||||
import os.path as osp
|
||||
|
||||
import torch
|
||||
import wandb
|
||||
import yaml
|
||||
from criterions import build_criterion
|
||||
from model import build_model
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
from trainer import build_eval, build_trainer
|
||||
from utils import *
|
||||
from utils import get_logger
|
||||
|
||||
from datasets import build_data_loader
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=str)
|
||||
parser.add_argument("--eval", action="store_true")
|
||||
parser.add_argument("--ckpt", type=str)
|
||||
parser.add_argument("--amp", action="store_true")
|
||||
parser.add_argument("--tensorboard", action="store_true")
|
||||
parser.add_argument("--wandb", action="store_true")
|
||||
parser.add_argument("--lr_scheduler", action="store_true")
|
||||
parser.add_argument("--no_rgb", action="store_true")
|
||||
parser.add_argument("--no_flow", action="store_true")
|
||||
parser.add_argument(
|
||||
"--save-as-cfg",
|
||||
action="store_true",
|
||||
help="Save outputs with config name in the output directory",
|
||||
)
|
||||
parser.add_argument("--debug", action="store_true")
|
||||
parser.add_argument(
|
||||
"--save-preds",
|
||||
action="store_true",
|
||||
help="Save predictions in the output directory",
|
||||
)
|
||||
# parser.add_argument('--model', type=str, default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
# combine argparse and yaml
|
||||
opt = yaml.load(open(args.config), Loader=yaml.FullLoader)
|
||||
opt.update(vars(args))
|
||||
cfg = opt
|
||||
|
||||
if "checkpoint" in cfg and cfg["checkpoint"] is not None and args.ckpt is None:
|
||||
args.ckpt = cfg["checkpoint"]
|
||||
print(f"checkpoint is given in the config, set args.ckpt to {args.ckpt}")
|
||||
|
||||
seed = 20 if "seed" not in cfg else cfg["seed"]
|
||||
set_seed(seed)
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
annotation_type = os.path.basename(cfg["annotation_type"])
|
||||
|
||||
identifier = os.path.basename(cfg["config"])[:-5]
|
||||
identifier += "_"
|
||||
if cfg["no_flow"] == False:
|
||||
identifier += "A"
|
||||
if cfg["no_rgb"] == False:
|
||||
identifier += "V"
|
||||
|
||||
if cfg["save_as_cfg"] == True:
|
||||
output_dir = osp.join("output_cfg", identifier)
|
||||
result_path = create_outdir(output_dir, overwrite=True)
|
||||
else:
|
||||
identifier = get_identifier(cfg)
|
||||
annotation_type = os.path.basename(cfg["annotation_type"])
|
||||
identifier += f"_{annotation_type}"
|
||||
result_path = create_outdir(osp.join(cfg["output_path"], identifier))
|
||||
|
||||
logger = get_logger(result_path)
|
||||
logger.info(f"Identifier: {identifier}")
|
||||
logger.info(f"result_path: {result_path}")
|
||||
logger.info(cfg)
|
||||
|
||||
testloader = build_data_loader(cfg, mode="test")
|
||||
print("testloader loaded")
|
||||
model = build_model(cfg, device)
|
||||
evaluate = build_eval(cfg)
|
||||
|
||||
trainloader = build_data_loader(cfg, mode="train")
|
||||
print("trainloader loaded")
|
||||
|
||||
# evaluation
|
||||
if args.eval == True and cfg["ckpt"] == None:
|
||||
print("Automatically find the best checkpoint")
|
||||
ckpt_root = os.path.join("output_cfg", identifier, "ckpts")
|
||||
ckpt_dirs = os.listdir(ckpt_root)
|
||||
|
||||
ckpt_path = os.path.join(ckpt_root, "best.pth")
|
||||
if os.path.exists(ckpt_path):
|
||||
args.ckpt = ckpt_path
|
||||
print(f"args.ckpt set to {args.ckpt}")
|
||||
|
||||
else:
|
||||
ckpt_path = os.path.join(ckpt_root, "pred", "best.pth")
|
||||
if os.path.exists(ckpt_path):
|
||||
args.ckpt = ckpt_path
|
||||
print(f"args.ckpt set to {args.ckpt}")
|
||||
|
||||
print("Warninig) no ckpt_path found, there can be multiple ckpts")
|
||||
print(f"ckpt_dirs: {ckpt_dirs}")
|
||||
pred_dir_number = 0
|
||||
ckpt_dirs.sort(reverse=True)
|
||||
|
||||
if args.ckpt == None:
|
||||
for ckpt_dir in ckpt_dirs:
|
||||
ckpt_path = os.path.join(ckpt_root, ckpt_dir, "best.pth")
|
||||
|
||||
if os.path.exists(ckpt_path):
|
||||
args.ckpt = ckpt_path
|
||||
print(f"Found ckpt, args.ckpt was None, set to {args.ckpt}")
|
||||
break
|
||||
|
||||
if args.ckpt == None:
|
||||
raise ValueError("No checkpoint found, args.ckpt is None")
|
||||
|
||||
# load checkpoint if args.ckpt is given
|
||||
if args.eval == True:
|
||||
if args.ckpt == None:
|
||||
raise ValueError("Please specify the checkpoint path")
|
||||
checkpoint = torch.load(args.ckpt)
|
||||
model.load_state_dict(checkpoint)
|
||||
|
||||
elif args.eval == False and args.ckpt != None:
|
||||
checkpoint = torch.load(args.ckpt)
|
||||
print(f"Load checkpoint from {args.ckpt}")
|
||||
model_dict = model.state_dict()
|
||||
|
||||
unnecessary_keys = []
|
||||
matched_ckpt = {}
|
||||
# Filter out unnecessary keys
|
||||
for k, v in checkpoint.items():
|
||||
if not (k in model_dict and model_dict[k].shape == checkpoint[k].shape):
|
||||
unnecessary_keys.append(k)
|
||||
elif (
|
||||
"except_classifier" in cfg
|
||||
and cfg["except_classifier"] == True
|
||||
and "classification" in k
|
||||
):
|
||||
print(f"Unnecessary key: {k} is classifier")
|
||||
unnecessary_keys.append(k)
|
||||
else:
|
||||
matched_ckpt[k] = v
|
||||
|
||||
# Overwrite entries in the existing state dict
|
||||
model_dict.update(matched_ckpt)
|
||||
|
||||
# Load the new state dict
|
||||
model.load_state_dict(model_dict)
|
||||
|
||||
print(f"Unnecessary keys: {unnecessary_keys}")
|
||||
|
||||
if cfg["wandb"] == True:
|
||||
# args.tensorboard = True
|
||||
print("wandb init")
|
||||
os.environ["WANDB__SERVICE_WAIT"] = "3000"
|
||||
wandb.init(
|
||||
project="egospeak",
|
||||
tags=[cfg["model"], cfg["data_name"], annotation_type],
|
||||
config=cfg,
|
||||
name=os.path.basename(result_path),
|
||||
)
|
||||
|
||||
if args.eval == True or args.debug == True:
|
||||
if args.eval == True and args.ckpt == None:
|
||||
raise ValueError("Please specify the checkpoint path")
|
||||
|
||||
logger.info(f'Dataset: {cfg["data_name"]}, Model: {cfg["model"]}')
|
||||
logger.info(f"Result path:{result_path}")
|
||||
|
||||
for phase in ["test"]:
|
||||
if phase == "train":
|
||||
raise NotImplementedError("evaluation on train split is not implemented")
|
||||
dataloader = trainloader
|
||||
|
||||
elif phase == "test":
|
||||
dataloader = testloader
|
||||
|
||||
mAP, mcAP, preds = evaluate(
|
||||
model, dataloader, logger, result_path=result_path
|
||||
)
|
||||
|
||||
pred_path = osp.join(result_path, "eval")
|
||||
os.makedirs(pred_path, exist_ok=True)
|
||||
|
||||
if args.save_preds == True:
|
||||
print(f"pred_path: {pred_path}")
|
||||
for k, v in preds.items():
|
||||
np.save(osp.join(pred_path, f"{k}.npy"), v)
|
||||
|
||||
logger.info(f'{phase}) {cfg["task"]} result: {mAP*100:.2f} mAP')
|
||||
logger.info(f'{phase}) {cfg["task"]} result: {mcAP*100:.2f} mcAP')
|
||||
exit()
|
||||
|
||||
criterion = build_criterion(cfg, device)
|
||||
train_one_epoch = build_trainer(cfg)
|
||||
optim = torch.optim.AdamW if cfg["optimizer"] == "AdamW" else torch.optim.Adam
|
||||
optimizer = optim(
|
||||
[{"params": model.parameters(), "initial_lr": cfg["lr"]}],
|
||||
lr=cfg["lr"],
|
||||
weight_decay=cfg["weight_decay"],
|
||||
)
|
||||
|
||||
scheduler = (
|
||||
build_lr_scheduler(cfg, optimizer, len(trainloader))
|
||||
if args.lr_scheduler
|
||||
else None
|
||||
)
|
||||
writer = SummaryWriter(osp.join(result_path, "runs")) if args.tensorboard else None
|
||||
scaler = torch.cuda.amp.GradScaler() if args.amp else None
|
||||
total_params = sum(p.numel() for p in model.parameters())
|
||||
|
||||
logger.info(f'Dataset: {cfg["data_name"]}, Model: {cfg["model"]}')
|
||||
logger.info(
|
||||
f'lr:{cfg["lr"]} | Weight Decay:{cfg["weight_decay"]} | Window Size:{cfg["window_size"]} | Batch Size:{cfg["batch_size"]}'
|
||||
)
|
||||
logger.info(
|
||||
f'Total epoch:{cfg["num_epoch"]} | Total Params:{total_params/1e6:.1f} M | Optimizer: {cfg["optimizer"]}'
|
||||
)
|
||||
logger.info(f"Output Path:{result_path}")
|
||||
|
||||
best_mAP, best_epoch = 0, 0
|
||||
best_cAP, best_cAP_epoch = 0, 0
|
||||
|
||||
pred_save_path = osp.join(result_path, "ckpts", "pred")
|
||||
print(f"pred_save_path: {pred_save_path}")
|
||||
|
||||
# if you want to overwrite, change the code
|
||||
os.makedirs(pred_save_path, exist_ok=True)
|
||||
|
||||
for epoch in range(1, cfg["num_epoch"] + 1):
|
||||
epoch_loss = train_one_epoch(
|
||||
trainloader,
|
||||
model,
|
||||
criterion,
|
||||
optimizer,
|
||||
scaler,
|
||||
epoch,
|
||||
cfg["wandb"],
|
||||
writer,
|
||||
scheduler=scheduler,
|
||||
)
|
||||
trainloader.dataset._init_features()
|
||||
|
||||
mAP, cAP, preds = evaluate(
|
||||
model, testloader, logger, criterion, cfg["wandb"], epoch, writer
|
||||
)
|
||||
|
||||
if mAP > best_mAP:
|
||||
best_mAP = mAP
|
||||
best_epoch = epoch
|
||||
torch.save(model.state_dict(), osp.join(pred_save_path, "best.pth"))
|
||||
|
||||
if args.save_preds == True:
|
||||
for k, v in preds.items():
|
||||
np.save(osp.join(pred_save_path, f"{k}.npy"), v)
|
||||
|
||||
if cAP > best_cAP:
|
||||
best_cAP = cAP
|
||||
best_cAP_epoch = epoch
|
||||
torch.save(model.state_dict(), osp.join(pred_save_path, "best_cAP.pth"))
|
||||
|
||||
logger.info(
|
||||
f'Epoch {epoch} mAP: {mAP*100:.2f} | Best mAP: {best_mAP*100:.2f} at epoch {best_epoch}, iter {epoch*cfg["batch_size"]*len(trainloader)} | train_loss: {epoch_loss/len(trainloader):.4f}, lr: {optimizer.param_groups[0]["lr"]:.7f}'
|
||||
)
|
||||
logger.info(
|
||||
f'Epoch {epoch} cAP: {cAP*100:.2f} | Best cAP: {best_cAP*100:.2f} at epoch {best_epoch}, iter {epoch*cfg["batch_size"]*len(trainloader)} | train_loss: {epoch_loss/len(trainloader):.4f}, lr: {optimizer.param_groups[0]["lr"]:.7f}'
|
||||
)
|
||||
|
||||
|
||||
if cfg["wandb"] == True:
|
||||
logger.info("Wandb logging")
|
||||
wandb.log(
|
||||
{
|
||||
"Epoch": epoch,
|
||||
"Train Loss (Epoch)": epoch_loss / len(trainloader),
|
||||
"Learning Rate": optimizer.param_groups[0]["lr"],
|
||||
"anticipation average mAP": mAP,
|
||||
"anticipation average cAP": cAP,
|
||||
}
|
||||
)
|
||||
|
||||
# os.rename(osp.join(result_path, 'ckpts', 'best.pth'), osp.join(result_path, 'ckpts', f'best_{best_mAP*100:.2f}.pth'))
|
||||
# save the training result and config as txt file
|
||||
with open(osp.join(result_path, "result.txt"), "w") as f:
|
||||
f.write(f"Best mAP: {best_mAP*100:.2f} at epoch {best_epoch}\n")
|
||||
f.write(f"Best cAP: {best_cAP*100:.2f} at epoch {best_cAP_epoch}\n")
|
||||
f.write(f"Config: {cfg}")
|
||||
|
||||
wandb.finish() if args.wandb else None
|
||||
4
third_party/EgoSpeak/MiniROAD/model/__init__.py
vendored
Executable file
4
third_party/EgoSpeak/MiniROAD/model/__init__.py
vendored
Executable file
@@ -0,0 +1,4 @@
|
||||
from .mamba import OADMamba
|
||||
from .model_builder import build_model
|
||||
from .rnn import MROAD
|
||||
from .transformer_models import ViTEnc
|
||||
1
third_party/EgoSpeak/MiniROAD/model/mamba/__init__.py
vendored
Normal file
1
third_party/EgoSpeak/MiniROAD/model/mamba/__init__.py
vendored
Normal file
@@ -0,0 +1 @@
|
||||
from .mamba import OADMamba, OADMambaA
|
||||
172
third_party/EgoSpeak/MiniROAD/model/mamba/mamba.py
vendored
Normal file
172
third_party/EgoSpeak/MiniROAD/model/mamba/mamba.py
vendored
Normal file
@@ -0,0 +1,172 @@
|
||||
import sys
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from configs.feature_sizes import FEATURE_SIZES
|
||||
from mamba_ssm import Mamba
|
||||
# from mambapy.mamba import Mamba, MambaConfig
|
||||
from model.model_builder import META_ARCHITECTURES
|
||||
from utils.util import parse_flow_type, parse_rgb_type
|
||||
|
||||
|
||||
@META_ARCHITECTURES.register("Mamba")
|
||||
class OADMamba(nn.Module):
|
||||
|
||||
def __init__(self, cfg):
|
||||
super(OADMamba, self).__init__()
|
||||
self.use_flow = not cfg['no_flow']
|
||||
self.use_rgb = not cfg['no_rgb']
|
||||
|
||||
self.input_dim = 0
|
||||
|
||||
rgb_type = parse_rgb_type(cfg['rgb_type'])
|
||||
flow_type = parse_flow_type(cfg['flow_type'])
|
||||
|
||||
if self.use_rgb:
|
||||
self.input_dim += FEATURE_SIZES[rgb_type]
|
||||
if self.use_flow:
|
||||
self.input_dim += FEATURE_SIZES[flow_type]
|
||||
|
||||
self.hidden_dim = cfg['hidden_dim']
|
||||
self.num_layers = cfg['num_layers']
|
||||
|
||||
# raise NotImplementedError("The following code is not implemented yet")
|
||||
self.out_dim = cfg['num_classes']
|
||||
self.window_size = cfg['window_size']
|
||||
|
||||
self.relu = nn.ReLU()
|
||||
|
||||
self.embedding_dim = cfg['embedding_dim']
|
||||
self.d_state = cfg['d_state']
|
||||
self.d_conv = cfg['d_conv']
|
||||
self.expand = cfg['expand']
|
||||
|
||||
# self.mamba_config = MambaConfig(d_model=self.embedding_dim, n_layers=self.num_layers)
|
||||
# self.mamba = Mamba(self.mamba_config)
|
||||
self.mamba = Mamba(
|
||||
d_model=self.embedding_dim, # Model dimension d_model
|
||||
d_state=self.d_state, # SSM state expansion factor
|
||||
d_conv=self.d_conv, # Local convolution width
|
||||
expand=self.expand, # Block expansion factor
|
||||
)
|
||||
|
||||
# self.gru = nn.GRU(self.embedding_dim, self.hidden_dim, self.num_layers, batch_first=True)
|
||||
self.layer1 = nn.Sequential(
|
||||
nn.Linear(self.input_dim, self.embedding_dim),
|
||||
nn.LayerNorm(self.embedding_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(p=cfg['dropout']),
|
||||
)
|
||||
self.f_classification = nn.Sequential(
|
||||
nn.Linear(self.hidden_dim, self.out_dim)
|
||||
)
|
||||
# self.h0 = torch.nn.Parameter(torch.zeros(1, 1, self.hidden_dim))
|
||||
self.h0 = torch.zeros(self.num_layers, 1, self.hidden_dim)
|
||||
|
||||
def forward(self, rgb_input, flow_input):
|
||||
x = torch.empty(0, device='cuda:0')
|
||||
if self.use_rgb:
|
||||
x = torch.cat((x, rgb_input), dim=2) # Concatenating along the channel dimension
|
||||
if self.use_flow:
|
||||
x = torch.cat((x, flow_input), dim=2)
|
||||
|
||||
x = self.layer1(x)
|
||||
ht = self.mamba(x)
|
||||
|
||||
logits = self.f_classification(ht)
|
||||
out_dict = {}
|
||||
if self.training:
|
||||
out_dict['logits'] = logits
|
||||
else:
|
||||
pred_scores = F.softmax(logits, dim=-1)
|
||||
out_dict['logits'] = pred_scores
|
||||
return out_dict
|
||||
|
||||
@META_ARCHITECTURES.register("MambaA")
|
||||
class OADMambaA(nn.Module):
|
||||
|
||||
def __init__(self, cfg):
|
||||
super(OADMambaA, self).__init__()
|
||||
self.use_flow = not cfg['no_flow']
|
||||
self.use_rgb = not cfg['no_rgb']
|
||||
self.input_dim = 0
|
||||
if self.use_rgb:
|
||||
self.input_dim += FEATURE_SIZES[cfg['rgb_type']]
|
||||
if self.use_flow:
|
||||
self.input_dim += FEATURE_SIZES[cfg['flow_type']]
|
||||
|
||||
self.hidden_dim = cfg['hidden_dim']
|
||||
self.num_layers = cfg['num_layers']
|
||||
self.anticipation_length = cfg["anticipation_length"]
|
||||
self.out_dim = cfg['num_classes']
|
||||
|
||||
self.embedding_dim = cfg['embedding_dim']
|
||||
self.d_state = cfg['d_state'] if 'd_state' in cfg else 16
|
||||
self.d_conv = cfg['d_conv'] if 'd_conv' in cfg else 4
|
||||
self.expand = cfg['expand'] if 'expand' in cfg else 2
|
||||
|
||||
self.layer1 = nn.Sequential(
|
||||
nn.Linear(self.input_dim, self.embedding_dim),
|
||||
nn.LayerNorm(self.embedding_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(p=cfg['dropout'])
|
||||
)
|
||||
self.actionness = cfg['actionness']
|
||||
if self.actionness:
|
||||
self.f_actionness = nn.Sequential(
|
||||
nn.Linear(self.hidden_dim, 1),
|
||||
)
|
||||
self.relu = nn.ReLU()
|
||||
# self.mamba_config = MambaConfig(d_model=self.embedding_dim, n_layers=self.num_layers)
|
||||
# self.mamba = Mamba(self.mamba_config)
|
||||
self.mamba = Mamba(
|
||||
d_model=self.embedding_dim, # Model dimension d_model
|
||||
d_state=self.d_state, # SSM state expansion factor
|
||||
d_conv=self.d_conv, # Local convolution width
|
||||
expand=self.expand, # Block expansion facto
|
||||
)
|
||||
|
||||
# self.gru = nn.GRU(self.embedding_dim, self.hidden_dim, self.num_layers, batch_first=True)
|
||||
self.f_classification = nn.Sequential(
|
||||
nn.Linear(self.embedding_dim, self.out_dim)
|
||||
)
|
||||
self.anticipation_layer = nn.Sequential(
|
||||
nn.Linear(self.embedding_dim, self.anticipation_length*self.embedding_dim),
|
||||
)
|
||||
self.cfg = cfg
|
||||
|
||||
def forward(self, rgb_input, flow_input):
|
||||
x = torch.empty(0, device='cuda:0')
|
||||
if self.use_rgb:
|
||||
# [64, 128, 2048]
|
||||
x = torch.cat((x, rgb_input), dim=2) # Concatenating along the channel dimension
|
||||
if self.use_flow:
|
||||
# [64, 128, 5120]
|
||||
x = torch.cat((x, flow_input), dim=2)
|
||||
|
||||
# audio channel is too huge to directly concat...
|
||||
# to c
|
||||
|
||||
# B, L, D: mamba
|
||||
B, S, feat_dim = x.shape # batch, length, dim
|
||||
#
|
||||
x = self.layer1(x)
|
||||
ht = self.mamba(x)
|
||||
# h0 = torch.zeros(1, B, self.hidden_dim).to(x.device)
|
||||
# ht, _ = self.gru(x, h0)
|
||||
logits = self.f_classification(self.relu(ht))
|
||||
anticipation_ht = self.anticipation_layer(self.relu(ht)).view(B, S, self.anticipation_length, self.embedding_dim)
|
||||
anticipation_logits = self.f_classification(self.relu(anticipation_ht))
|
||||
|
||||
out_dict = {}
|
||||
if self.training:
|
||||
out_dict['logits'] = logits
|
||||
out_dict['anticipation_logits'] = anticipation_logits
|
||||
else:
|
||||
pred_scores = F.softmax(logits, dim=-1)
|
||||
pred_anticipation_scores = F.softmax(anticipation_logits, dim=-1)
|
||||
out_dict['logits'] = pred_scores
|
||||
out_dict['anticipation_logits'] = pred_anticipation_scores
|
||||
|
||||
return out_dict
|
||||
9
third_party/EgoSpeak/MiniROAD/model/model_builder.py
vendored
Executable file
9
third_party/EgoSpeak/MiniROAD/model/model_builder.py
vendored
Executable file
@@ -0,0 +1,9 @@
|
||||
__all__ = ['build_model']
|
||||
|
||||
from utils import Registry
|
||||
|
||||
META_ARCHITECTURES = Registry()
|
||||
|
||||
def build_model(cfg, device=None):
|
||||
model = META_ARCHITECTURES[cfg["model"]](cfg)
|
||||
return model.to(device)
|
||||
1
third_party/EgoSpeak/MiniROAD/model/rnn/__init__.py
vendored
Executable file
1
third_party/EgoSpeak/MiniROAD/model/rnn/__init__.py
vendored
Executable file
@@ -0,0 +1 @@
|
||||
from .rnn import MROAD, MROADA
|
||||
153
third_party/EgoSpeak/MiniROAD/model/rnn/rnn.py
vendored
Executable file
153
third_party/EgoSpeak/MiniROAD/model/rnn/rnn.py
vendored
Executable file
@@ -0,0 +1,153 @@
|
||||
import sys
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from configs.feature_sizes import FEATURE_SIZES
|
||||
from model.model_builder import META_ARCHITECTURES
|
||||
from utils.util import parse_flow_type, parse_rgb_type
|
||||
|
||||
|
||||
@META_ARCHITECTURES.register("MiniROAD")
|
||||
class MROAD(nn.Module):
|
||||
|
||||
def __init__(self, cfg):
|
||||
super(MROAD, self).__init__()
|
||||
self.use_flow = not cfg['no_flow']
|
||||
self.use_rgb = not cfg['no_rgb']
|
||||
|
||||
self.input_dim = 0
|
||||
|
||||
rgb_type = parse_rgb_type(cfg['rgb_type'])
|
||||
flow_type = parse_flow_type(cfg['flow_type'])
|
||||
|
||||
if self.use_rgb:
|
||||
self.input_dim += FEATURE_SIZES[rgb_type]
|
||||
if self.use_flow:
|
||||
self.input_dim += FEATURE_SIZES[flow_type]
|
||||
|
||||
self.hidden_dim = cfg['hidden_dim']
|
||||
self.num_layers = cfg['num_layers']
|
||||
self.out_dim = cfg['num_classes']
|
||||
self.window_size = cfg['window_size']
|
||||
|
||||
self.relu = nn.ReLU()
|
||||
self.embedding_dim = cfg['embedding_dim']
|
||||
self.gru = nn.GRU(self.embedding_dim, self.hidden_dim, self.num_layers, batch_first=True)
|
||||
self.layer1 = nn.Sequential(
|
||||
nn.Linear(self.input_dim, self.embedding_dim),
|
||||
nn.LayerNorm(self.embedding_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(p=cfg['dropout']),
|
||||
)
|
||||
self.f_classification = nn.Sequential(
|
||||
nn.Linear(self.hidden_dim, self.out_dim)
|
||||
)
|
||||
# self.h0 = torch.nn.Parameter(torch.zeros(1, 1, self.hidden_dim))
|
||||
self.h0 = torch.zeros(self.num_layers, 1, self.hidden_dim)
|
||||
|
||||
def forward(self, rgb_input, flow_input):
|
||||
x = torch.empty(0, device='cuda:0')
|
||||
if self.use_rgb:
|
||||
x = torch.cat((x, rgb_input), dim=2) # Concatenating along the channel dimension
|
||||
if self.use_flow:
|
||||
x = torch.cat((x, flow_input), dim=2)
|
||||
|
||||
# if self.use_rgb and self.use_flow:
|
||||
# x = torch.cat((rgb_input, flow_input), 2)
|
||||
# elif self.use_rgb:
|
||||
# x = rgb_input
|
||||
# elif self.use_flow:
|
||||
# x = flow_input
|
||||
|
||||
x = self.layer1(x)
|
||||
B, _, _ = x.shape
|
||||
h0 = self.h0.expand(-1, B, -1).to(x.device)
|
||||
ht, _ = self.gru(x, h0)
|
||||
ht = self.relu(ht)
|
||||
# ht = self.relu(ht + x)
|
||||
logits = self.f_classification(ht)
|
||||
out_dict = {}
|
||||
if self.training:
|
||||
out_dict['logits'] = logits
|
||||
else:
|
||||
pred_scores = F.softmax(logits, dim=-1)
|
||||
out_dict['logits'] = pred_scores
|
||||
return out_dict
|
||||
|
||||
@META_ARCHITECTURES.register("MiniROADA")
|
||||
class MROADA(nn.Module):
|
||||
|
||||
def __init__(self, cfg):
|
||||
super(MROADA, self).__init__()
|
||||
self.use_flow = not cfg['no_flow']
|
||||
self.use_rgb = not cfg['no_rgb']
|
||||
self.input_dim = 0
|
||||
if self.use_rgb:
|
||||
self.input_dim += FEATURE_SIZES[cfg['rgb_type']]
|
||||
if self.use_flow:
|
||||
self.input_dim += FEATURE_SIZES[cfg['flow_type']]
|
||||
|
||||
self.embedding_dim = cfg['embedding_dim']
|
||||
self.hidden_dim = cfg['hidden_dim']
|
||||
self.num_layers = cfg['num_layers']
|
||||
self.anticipation_length = cfg["anticipation_length"]
|
||||
self.out_dim = cfg['num_classes']
|
||||
|
||||
self.layer1 = nn.Sequential(
|
||||
nn.Linear(self.input_dim, self.embedding_dim),
|
||||
nn.LayerNorm(self.embedding_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(p=cfg['dropout'])
|
||||
)
|
||||
|
||||
self.actionness = cfg['actionness']
|
||||
if self.actionness:
|
||||
self.f_actionness = nn.Sequential(
|
||||
nn.Linear(self.hidden_dim, 1),
|
||||
)
|
||||
self.relu = nn.ReLU()
|
||||
self.gru = nn.GRU(self.embedding_dim, self.hidden_dim, self.num_layers, batch_first=True)
|
||||
self.f_classification = nn.Sequential(
|
||||
nn.Linear(self.hidden_dim, self.out_dim)
|
||||
)
|
||||
self.anticipation_layer = nn.Sequential(
|
||||
nn.Linear(self.hidden_dim, self.anticipation_length*self.hidden_dim),
|
||||
)
|
||||
self.cfg = cfg
|
||||
|
||||
def forward(self, rgb_input, flow_input):
|
||||
x = torch.empty(0, device='cuda:0')
|
||||
if self.use_rgb:
|
||||
# [64, 128, 2048]
|
||||
x = torch.cat((x, rgb_input), dim=2) # Concatenating along the channel dimension
|
||||
if self.use_flow:
|
||||
# [64, 128, 5120]
|
||||
x = torch.cat((x, flow_input), dim=2)
|
||||
|
||||
# audio channel is too huge to directly concat...
|
||||
# to c
|
||||
|
||||
B, S, feat_dim = x.shape # batch, length, dim
|
||||
#
|
||||
x = self.layer1(x)
|
||||
h0 = torch.zeros(1, B, self.hidden_dim).to(x.device)
|
||||
ht, _ = self.gru(x, h0)
|
||||
ht = self.relu(ht)
|
||||
|
||||
logits = self.f_classification(self.relu(ht))
|
||||
anticipation_ht = self.anticipation_layer(self.relu(ht)).view(B, S, self.anticipation_length, self.hidden_dim)
|
||||
anticipation_logits = self.f_classification(self.relu(anticipation_ht))
|
||||
|
||||
|
||||
out_dict = {}
|
||||
if self.training:
|
||||
out_dict['logits'] = logits
|
||||
out_dict['anticipation_logits'] = anticipation_logits
|
||||
else:
|
||||
pred_scores = F.softmax(logits, dim=-1)
|
||||
pred_anticipation_scores = F.softmax(anticipation_logits, dim=-1)
|
||||
out_dict['logits'] = pred_scores
|
||||
out_dict['anticipation_logits'] = pred_anticipation_scores
|
||||
|
||||
return out_dict
|
||||
155
third_party/EgoSpeak/MiniROAD/model/transformer_models/Attention.py
vendored
Executable file
155
third_party/EgoSpeak/MiniROAD/model/transformer_models/Attention.py
vendored
Executable file
@@ -0,0 +1,155 @@
|
||||
import math
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class SelfAttention(nn.Module):
|
||||
def __init__(
|
||||
self, dim, heads=8, qkv_bias=False, qk_scale=None, dropout_rate=0.0
|
||||
):
|
||||
super().__init__()
|
||||
self.num_heads = heads
|
||||
head_dim = dim // heads
|
||||
self.scale = qk_scale or head_dim ** -0.5
|
||||
|
||||
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
|
||||
self.attn_drop = nn.Dropout(dropout_rate)
|
||||
self.proj = nn.Linear(dim, dim)
|
||||
self.proj_drop = nn.Dropout(dropout_rate)
|
||||
|
||||
def forward(self, x):
|
||||
B, N, C = x.shape
|
||||
qkv = (
|
||||
self.qkv(x)
|
||||
.reshape(B, N, 3, self.num_heads, C // self.num_heads)
|
||||
.permute(2, 0, 3, 1, 4)
|
||||
)
|
||||
q, k, v = (
|
||||
qkv[0],
|
||||
qkv[1],
|
||||
qkv[2],
|
||||
) # make torchscript happy (cannot use tensor as tuple)
|
||||
|
||||
attn = (q @ k.transpose(-2, -1)) * self.scale
|
||||
attn = attn.softmax(dim=-1)
|
||||
attn = self.attn_drop(attn)
|
||||
|
||||
x = (attn @ v).transpose(1, 2).reshape(B, N, C)
|
||||
x = self.proj(x)
|
||||
x = self.proj_drop(x)
|
||||
return x
|
||||
|
||||
|
||||
class AxialAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_planes,
|
||||
out_planes,
|
||||
groups=8,
|
||||
kernel_size=56,
|
||||
stride=1,
|
||||
bias=False,
|
||||
width=False,
|
||||
):
|
||||
assert (in_planes % groups == 0) and (out_planes % groups == 0)
|
||||
super(AxialAttention, self).__init__()
|
||||
self.in_planes = in_planes
|
||||
self.out_planes = out_planes
|
||||
self.groups = groups
|
||||
self.group_planes = out_planes // groups
|
||||
self.kernel_size = kernel_size
|
||||
self.stride = stride
|
||||
self.bias = bias
|
||||
self.width = width
|
||||
|
||||
# Multi-head self attention
|
||||
self.qkv_transform = nn.Conv1d(
|
||||
in_planes,
|
||||
out_planes * 2,
|
||||
kernel_size=1,
|
||||
stride=1,
|
||||
padding=0,
|
||||
bias=False,
|
||||
)
|
||||
self.bn_qkv = nn.BatchNorm1d(out_planes * 2)
|
||||
self.bn_similarity = nn.BatchNorm2d(groups * 3)
|
||||
self.bn_output = nn.BatchNorm1d(out_planes * 2)
|
||||
|
||||
# Position embedding
|
||||
self.relative = nn.Parameter(
|
||||
torch.randn(self.group_planes * 2, kernel_size * 2 - 1),
|
||||
requires_grad=True,
|
||||
)
|
||||
query_index = torch.arange(kernel_size).unsqueeze(0)
|
||||
key_index = torch.arange(kernel_size).unsqueeze(1)
|
||||
relative_index = key_index - query_index + kernel_size - 1
|
||||
self.register_buffer('flatten_index', relative_index.view(-1))
|
||||
if stride > 1:
|
||||
self.pooling = nn.AvgPool2d(stride, stride=stride)
|
||||
|
||||
self.reset_parameters()
|
||||
|
||||
def forward(self, x):
|
||||
if self.width:
|
||||
x = x.permute(0, 2, 1, 3)
|
||||
else:
|
||||
x = x.permute(0, 3, 1, 2) # N, W, C, H
|
||||
N, W, C, H = x.shape
|
||||
x = x.contiguous().view(N * W, C, H)
|
||||
|
||||
# Transformations
|
||||
qkv = self.bn_qkv(self.qkv_transform(x))
|
||||
q, k, v = torch.split(
|
||||
qkv.reshape(N * W, self.groups, self.group_planes * 2, H),
|
||||
[self.group_planes // 2, self.group_planes // 2, self.group_planes],
|
||||
dim=2,
|
||||
)
|
||||
|
||||
# Calculate position embedding
|
||||
all_embeddings = torch.index_select(
|
||||
self.relative, 1, self.flatten_index
|
||||
).view(self.group_planes * 2, self.kernel_size, self.kernel_size)
|
||||
q_embedding, k_embedding, v_embedding = torch.split(
|
||||
all_embeddings,
|
||||
[self.group_planes // 2, self.group_planes // 2, self.group_planes],
|
||||
dim=0,
|
||||
)
|
||||
qr = torch.einsum('bgci,cij->bgij', q, q_embedding)
|
||||
kr = torch.einsum('bgci,cij->bgij', k, k_embedding).transpose(2, 3)
|
||||
qk = torch.einsum('bgci, bgcj->bgij', q, k)
|
||||
stacked_similarity = torch.cat([qk, qr, kr], dim=1)
|
||||
stacked_similarity = (
|
||||
self.bn_similarity(stacked_similarity)
|
||||
.view(N * W, 3, self.groups, H, H)
|
||||
.sum(dim=1)
|
||||
)
|
||||
|
||||
# (N, groups, H, H, W)
|
||||
similarity = F.softmax(stacked_similarity, dim=3)
|
||||
sv = torch.einsum('bgij,bgcj->bgci', similarity, v)
|
||||
sve = torch.einsum('bgij,cij->bgci', similarity, v_embedding)
|
||||
stacked_output = torch.cat([sv, sve], dim=-1).view(
|
||||
N * W, self.out_planes * 2, H
|
||||
)
|
||||
output = (
|
||||
self.bn_output(stacked_output)
|
||||
.view(N, W, self.out_planes, 2, H)
|
||||
.sum(dim=-2)
|
||||
)
|
||||
|
||||
if self.width:
|
||||
output = output.permute(0, 2, 1, 3)
|
||||
else:
|
||||
output = output.permute(0, 2, 3, 1)
|
||||
|
||||
if self.stride > 1:
|
||||
output = self.pooling(output)
|
||||
|
||||
return output
|
||||
|
||||
def reset_parameters(self):
|
||||
self.qkv_transform.weight.data.normal_(
|
||||
0, math.sqrt(1.0 / self.in_planes)
|
||||
)
|
||||
nn.init.normal_(self.relative, 0.0, math.sqrt(1.0 / self.group_planes))
|
||||
244
third_party/EgoSpeak/MiniROAD/model/transformer_models/AxialNet.py
vendored
Executable file
244
third_party/EgoSpeak/MiniROAD/model/transformer_models/AxialNet.py
vendored
Executable file
@@ -0,0 +1,244 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from .Attention import AxialAttention
|
||||
|
||||
|
||||
def conv1x1(in_planes, out_planes, stride=1):
|
||||
"""1x1 convolution"""
|
||||
return nn.Conv2d(
|
||||
in_planes, out_planes, kernel_size=1, stride=stride, bias=False
|
||||
)
|
||||
|
||||
|
||||
class AxialBlock(nn.Module):
|
||||
expansion = 2
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
inplanes,
|
||||
planes,
|
||||
stride=1,
|
||||
downsample=None,
|
||||
groups=1,
|
||||
base_width=64,
|
||||
dilation=1,
|
||||
norm_layer=None,
|
||||
kernel_size=56,
|
||||
):
|
||||
super(AxialBlock, self).__init__()
|
||||
if norm_layer is None:
|
||||
norm_layer = nn.BatchNorm2d
|
||||
width = int(planes * (base_width / 64.0))
|
||||
# Both self.conv2 and self.downsample layers downsample the input when stride != 1
|
||||
self.conv_down = conv1x1(inplanes, width)
|
||||
self.bn1 = norm_layer(width)
|
||||
self.hight_block = AxialAttention(
|
||||
width, width, groups=groups, kernel_size=kernel_size
|
||||
)
|
||||
self.width_block = AxialAttention(
|
||||
width,
|
||||
width,
|
||||
groups=groups,
|
||||
kernel_size=kernel_size,
|
||||
stride=stride,
|
||||
width=True,
|
||||
)
|
||||
self.conv_up = conv1x1(width, planes * self.expansion)
|
||||
self.bn2 = norm_layer(planes * self.expansion)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
|
||||
def forward(self, x):
|
||||
identity = x
|
||||
|
||||
out = self.conv_down(x)
|
||||
out = self.bn1(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.hight_block(out)
|
||||
out = self.width_block(out)
|
||||
out = self.relu(out)
|
||||
|
||||
out = self.conv_up(out)
|
||||
out = self.bn2(out)
|
||||
|
||||
if self.downsample is not None:
|
||||
identity = self.downsample(x)
|
||||
|
||||
out += identity
|
||||
out = self.relu(out)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
class AxialAttentionNet(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
layers,
|
||||
num_classes=1000,
|
||||
zero_init_residual=True,
|
||||
groups=8,
|
||||
width_per_group=64,
|
||||
replace_stride_with_dilation=None,
|
||||
norm_layer=None,
|
||||
s=0.5,
|
||||
):
|
||||
super(AxialAttentionNet, self).__init__()
|
||||
block = AxialBlock
|
||||
if norm_layer is None:
|
||||
norm_layer = nn.BatchNorm2d
|
||||
self._norm_layer = norm_layer
|
||||
|
||||
self.inplanes = int(64 * s)
|
||||
self.dilation = 1
|
||||
if replace_stride_with_dilation is None:
|
||||
# each element in the tuple indicates if we should replace
|
||||
# the 2x2 stride with a dilated convolution instead
|
||||
replace_stride_with_dilation = [False, False, False]
|
||||
if len(replace_stride_with_dilation) != 3:
|
||||
raise ValueError(
|
||||
"replace_stride_with_dilation should be None "
|
||||
"or a 3-element tuple, got {}".format(
|
||||
replace_stride_with_dilation
|
||||
)
|
||||
)
|
||||
self.groups = groups
|
||||
self.base_width = width_per_group
|
||||
self.conv1 = nn.Conv2d(
|
||||
3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False
|
||||
)
|
||||
self.bn1 = norm_layer(self.inplanes)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
|
||||
self.layer1 = self._make_layer(
|
||||
block, int(128 * s), layers[0], kernel_size=56
|
||||
)
|
||||
self.layer2 = self._make_layer(
|
||||
block,
|
||||
int(256 * s),
|
||||
layers[1],
|
||||
stride=2,
|
||||
kernel_size=56,
|
||||
dilate=replace_stride_with_dilation[0],
|
||||
)
|
||||
self.layer3 = self._make_layer(
|
||||
block,
|
||||
int(512 * s),
|
||||
layers[2],
|
||||
stride=2,
|
||||
kernel_size=28,
|
||||
dilate=replace_stride_with_dilation[1],
|
||||
)
|
||||
self.layer4 = self._make_layer(
|
||||
block,
|
||||
int(1024 * s),
|
||||
layers[3],
|
||||
stride=2,
|
||||
kernel_size=14,
|
||||
dilate=replace_stride_with_dilation[2],
|
||||
)
|
||||
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
|
||||
self.fc = nn.Linear(int(1024 * block.expansion * s), num_classes)
|
||||
|
||||
for m in self.modules():
|
||||
if isinstance(m, (nn.Conv2d, nn.Conv1d)):
|
||||
if isinstance(m, nn.Conv1d):
|
||||
pass
|
||||
else:
|
||||
nn.init.kaiming_normal_(
|
||||
m.weight, mode='fan_out', nonlinearity='relu'
|
||||
)
|
||||
elif isinstance(m, (nn.BatchNorm2d, nn.BatchNorm1d, nn.GroupNorm)):
|
||||
nn.init.constant_(m.weight, 1)
|
||||
nn.init.constant_(m.bias, 0)
|
||||
|
||||
# Zero-initialize the last BN in each residual branch,
|
||||
# so that the residual branch starts with zeros, and each residual block behaves like an identity.
|
||||
# This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
|
||||
if zero_init_residual:
|
||||
for m in self.modules():
|
||||
if isinstance(m, AxialBlock):
|
||||
nn.init.constant_(m.bn2.weight, 0)
|
||||
|
||||
def _make_layer(
|
||||
self, block, planes, blocks, kernel_size=56, stride=1, dilate=False
|
||||
):
|
||||
norm_layer = self._norm_layer
|
||||
downsample = None
|
||||
previous_dilation = self.dilation
|
||||
if dilate:
|
||||
self.dilation *= stride
|
||||
stride = 1
|
||||
if stride != 1 or self.inplanes != planes * block.expansion:
|
||||
downsample = nn.Sequential(
|
||||
conv1x1(self.inplanes, planes * block.expansion, stride),
|
||||
norm_layer(planes * block.expansion),
|
||||
)
|
||||
|
||||
layers = []
|
||||
layers.append(
|
||||
block(
|
||||
self.inplanes,
|
||||
planes,
|
||||
stride,
|
||||
downsample,
|
||||
groups=self.groups,
|
||||
base_width=self.base_width,
|
||||
dilation=previous_dilation,
|
||||
norm_layer=norm_layer,
|
||||
kernel_size=kernel_size,
|
||||
)
|
||||
)
|
||||
self.inplanes = planes * block.expansion
|
||||
if stride != 1:
|
||||
kernel_size = kernel_size // 2
|
||||
|
||||
for _ in range(1, blocks):
|
||||
layers.append(
|
||||
block(
|
||||
self.inplanes,
|
||||
planes,
|
||||
groups=self.groups,
|
||||
base_width=self.base_width,
|
||||
dilation=self.dilation,
|
||||
norm_layer=norm_layer,
|
||||
kernel_size=kernel_size,
|
||||
)
|
||||
)
|
||||
|
||||
return nn.Sequential(*layers)
|
||||
|
||||
def _forward_impl(self, x, include_conv5=False, include_top=False):
|
||||
# See note [TorchScript super()]
|
||||
x = self.conv1(x)
|
||||
x = self.bn1(x)
|
||||
x = self.relu(x)
|
||||
x = self.maxpool(x)
|
||||
|
||||
x = self.layer1(x)
|
||||
x = self.layer2(x)
|
||||
x = self.layer3(x)
|
||||
if include_conv5:
|
||||
x = self.layer4(x)
|
||||
|
||||
if include_top:
|
||||
x = self.avgpool(x)
|
||||
x = torch.flatten(x, 1)
|
||||
x = self.fc(x)
|
||||
|
||||
return x
|
||||
|
||||
def forward(self, x, include_conv5=False, include_top=False):
|
||||
return self._forward_impl(x, include_conv5, include_top)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
model = AxialAttentionNet([1, 2, 4, 1], s=0.5)
|
||||
# model = AxialAttentionNet([3, 4, 6, 3], s=0.25)
|
||||
# print(model)
|
||||
|
||||
x = torch.randn([8, 3, 224, 224])
|
||||
print(model(x).shape)
|
||||
print(model(x, include_conv5=True).shape)
|
||||
print(model(x, include_conv5=True, include_top=True).shape)
|
||||
138
third_party/EgoSpeak/MiniROAD/model/transformer_models/BiT.py
vendored
Executable file
138
third_party/EgoSpeak/MiniROAD/model/transformer_models/BiT.py
vendored
Executable file
@@ -0,0 +1,138 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
class StdConv2d(nn.Conv2d):
|
||||
def forward(self, x):
|
||||
w = self.weight
|
||||
v, m = torch.var_mean(w, dim=[1, 2, 3], keepdim=True, unbiased=False)
|
||||
w = (w - m) / torch.sqrt(v + 1e-10)
|
||||
return F.conv2d(
|
||||
x,
|
||||
w,
|
||||
self.bias,
|
||||
self.stride,
|
||||
self.padding,
|
||||
self.dilation,
|
||||
self.groups,
|
||||
)
|
||||
|
||||
|
||||
def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
|
||||
"""3x3 convolution with padding"""
|
||||
return StdConv2d(
|
||||
in_planes,
|
||||
out_planes,
|
||||
kernel_size=3,
|
||||
stride=stride,
|
||||
padding=dilation,
|
||||
groups=groups,
|
||||
bias=False,
|
||||
dilation=dilation,
|
||||
)
|
||||
|
||||
|
||||
def conv1x1(in_planes, out_planes, stride=1):
|
||||
"""1x1 convolution"""
|
||||
return StdConv2d(
|
||||
in_planes, out_planes, kernel_size=1, stride=stride, bias=False
|
||||
)
|
||||
|
||||
|
||||
class PreActBottleneck(nn.Module):
|
||||
"""Pre-activation (v2) bottleneck block.
|
||||
Follows the implementation of "Identity Mappings in Deep Residual Networks":
|
||||
https://github.com/KaimingHe/resnet-1k-layers/blob/master/resnet-pre-act.lua
|
||||
Except it puts the stride on 3x3 conv when available.
|
||||
"""
|
||||
|
||||
def __init__(self, in_planes, out_planes=None, mid_planes=None, stride=1):
|
||||
super(PreActBottleneck, self).__init__()
|
||||
out_planes = out_planes or in_planes
|
||||
mid_planes = mid_planes or out_planes // 4
|
||||
|
||||
self.gn1 = nn.GroupNorm(32, in_planes)
|
||||
self.conv1 = conv1x1(in_planes, mid_planes)
|
||||
self.gn2 = nn.GroupNorm(32, mid_planes)
|
||||
self.conv2 = conv3x3(mid_planes, mid_planes, stride)
|
||||
self.gn3 = nn.GroupNorm(32, mid_planes)
|
||||
self.conv3 = conv1x1(mid_planes, out_planes)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
|
||||
if stride != 1 or in_planes != out_planes:
|
||||
# Projection also with pre-activation according to paper.
|
||||
self.downsample = conv1x1(in_planes, out_planes, stride)
|
||||
|
||||
def forward(self, x):
|
||||
out = self.relu(self.gn1(x))
|
||||
|
||||
# Residual branch
|
||||
residual = x
|
||||
if hasattr(self, 'downsample'):
|
||||
residual = self.downsample(out)
|
||||
|
||||
# Unit's branch
|
||||
out = self.conv1(out)
|
||||
out = self.conv2(self.relu(self.gn2(out)))
|
||||
out = self.conv3(self.relu(self.gn3(out)))
|
||||
|
||||
return out + residual
|
||||
|
||||
|
||||
class ResNetV2Model(nn.Module):
|
||||
"""Implementation of Pre-activation (v2) ResNet mode."""
|
||||
|
||||
def __init__(self, block_units, width_factor, head_size=21843):
|
||||
super(ResNetV2Model, self).__init__()
|
||||
wf = width_factor # shortcut 'cause we'll use it a lot.
|
||||
|
||||
# The following will be unreadable if we split lines.
|
||||
# pylint: disable=line-too-long
|
||||
self.conv1 = nn.Sequential(OrderedDict([
|
||||
('conv', StdConv2d(3, 64*wf, kernel_size=7, stride=2, padding=3, bias=False)),
|
||||
('pad', nn.ConstantPad2d(1, 0)),
|
||||
('pool', nn.MaxPool2d(kernel_size=3, stride=2, padding=0)),
|
||||
]))
|
||||
|
||||
self.conv2 = nn.Sequential(OrderedDict(
|
||||
[('unit01', PreActBottleneck(in_planes=64*wf, out_planes=256*wf, mid_planes=64*wf))] +
|
||||
[(f'unit{i:02d}', PreActBottleneck(in_planes=256*wf, out_planes=256*wf, mid_planes=64*wf)) for i in range(2, block_units[0] + 1)],
|
||||
))
|
||||
self.conv3 = nn.Sequential(OrderedDict(
|
||||
[('unit01', PreActBottleneck(in_planes=256*wf, out_planes=512*wf, mid_planes=128*wf, stride=2))] +
|
||||
[(f'unit{i:02d}', PreActBottleneck(in_planes=512*wf, out_planes=512*wf, mid_planes=128*wf)) for i in range(2, block_units[1] + 1)],
|
||||
))
|
||||
self.conv4 = nn.Sequential(OrderedDict(
|
||||
[('unit01', PreActBottleneck(in_planes=512*wf, out_planes=1024*wf, mid_planes=256*wf, stride=2))] +
|
||||
[(f'unit{i:02d}', PreActBottleneck(in_planes=1024*wf, out_planes=1024*wf, mid_planes=256*wf)) for i in range(2, block_units[2] + 1)],
|
||||
))
|
||||
self.conv5 = nn.Sequential(OrderedDict(
|
||||
[('unit01', PreActBottleneck(in_planes=1024*wf, out_planes=2048*wf, mid_planes=512*wf, stride=2))] +
|
||||
[(f'unit{i:02d}', PreActBottleneck(in_planes=2048*wf, out_planes=2048*wf, mid_planes=512*wf)) for i in range(2, block_units[3] + 1)],
|
||||
))
|
||||
# pylint: enable=line-too-long
|
||||
|
||||
self.head = nn.Sequential(OrderedDict([
|
||||
('gn', nn.GroupNorm(32, 2048*wf)),
|
||||
('relu', nn.ReLU(inplace=True)),
|
||||
('avg', nn.AdaptiveAvgPool2d(output_size=1)),
|
||||
('conv', nn.Conv2d(2048*wf, head_size, kernel_size=1, bias=True)),
|
||||
]))
|
||||
|
||||
def forward(self, x, include_conv5=False, include_top=False):
|
||||
x = self.conv1(x)
|
||||
x = self.conv2(x)
|
||||
x = self.conv3(x)
|
||||
x = self.conv4(x)
|
||||
if include_conv5:
|
||||
x = self.conv5(x)
|
||||
if include_top:
|
||||
x = self.head(x)
|
||||
|
||||
if include_top and include_conv5:
|
||||
assert x.shape[-2:] == (1,1,)
|
||||
return x[..., 0, 0]
|
||||
|
||||
return x
|
||||
223
third_party/EgoSpeak/MiniROAD/model/transformer_models/HybridViT.py
vendored
Executable file
223
third_party/EgoSpeak/MiniROAD/model/transformer_models/HybridViT.py
vendored
Executable file
@@ -0,0 +1,223 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from .BiT import ResNetV2Model
|
||||
from .AxialNet import AxialAttentionNet
|
||||
from .Transformer import TransformerModel
|
||||
from .PositionalEncoding import (
|
||||
FixedPositionalEncoding,
|
||||
LearnedPositionalEncoding,
|
||||
)
|
||||
|
||||
|
||||
class HybridVisionTransformer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
img_dim,
|
||||
out_dim,
|
||||
num_channels,
|
||||
embedding_dim,
|
||||
num_heads,
|
||||
num_layers,
|
||||
hidden_dim,
|
||||
include_conv5,
|
||||
dropout_rate,
|
||||
positional_encoding_type,
|
||||
backbone=None,
|
||||
):
|
||||
super(HybridVisionTransformer, self).__init__()
|
||||
|
||||
assert embedding_dim % num_heads == 0
|
||||
|
||||
self.embedding_dim = embedding_dim
|
||||
self.num_heads = num_heads
|
||||
self.out_dim = out_dim
|
||||
self.num_channels = num_channels
|
||||
self.include_conv5 = include_conv5
|
||||
self.backbone = backbone
|
||||
|
||||
self.backbone_model, self.flatten_dim = self.configure_backbone()
|
||||
|
||||
self.projection_encoding = nn.Linear(self.flatten_dim, embedding_dim)
|
||||
self.cls_token = nn.Parameter(torch.randn(1, 1, embedding_dim))
|
||||
|
||||
self.decoder_dim = int(img_dim / 16.0) ** 2
|
||||
if self.include_conv5:
|
||||
self.decoder_dim = int(img_dim / 32.0) ** 2
|
||||
|
||||
self.decoder_dim += 1 # for the cls token
|
||||
|
||||
if positional_encoding_type == "learned":
|
||||
self.position_encoding = LearnedPositionalEncoding(
|
||||
self.decoder_dim, self.embedding_dim, self.decoder_dim
|
||||
)
|
||||
elif positional_encoding_type == "fixed":
|
||||
self.position_encoding = FixedPositionalEncoding(
|
||||
self.embedding_dim,
|
||||
)
|
||||
|
||||
self.transformer = TransformerModel(
|
||||
embedding_dim, num_layers, num_heads, hidden_dim
|
||||
)
|
||||
self.mlp_head = nn.Sequential(
|
||||
nn.Linear(embedding_dim, hidden_dim),
|
||||
nn.GELU(),
|
||||
nn.Linear(hidden_dim, out_dim),
|
||||
)
|
||||
self.to_cls_token = nn.Identity()
|
||||
|
||||
def forward(self, x):
|
||||
# apply bit backbone
|
||||
x = self.backbone_model(x, include_conv5=self.include_conv5)
|
||||
x = x.view(x.size(0), -1, self.flatten_dim)
|
||||
|
||||
x = self.projection_encoding(x)
|
||||
cls_tokens = self.cls_token.expand(x.shape[0], -1, -1)
|
||||
x = torch.cat((cls_tokens, x), dim=1)
|
||||
x = self.position_encoding(x)
|
||||
|
||||
# apply transformer
|
||||
x = self.transformer(x)
|
||||
x = self.to_cls_token(x[:, 0])
|
||||
x = self.mlp_head(x)
|
||||
x = F.log_softmax(x, dim=-1)
|
||||
|
||||
return x
|
||||
|
||||
def configure_backbone(self):
|
||||
raise NotImplementedError("Method to be called in child class!!")
|
||||
|
||||
|
||||
class ResNetHybridViT(HybridVisionTransformer):
|
||||
def __init__(
|
||||
self,
|
||||
img_dim,
|
||||
out_dim,
|
||||
num_channels,
|
||||
embedding_dim,
|
||||
num_heads,
|
||||
num_layers,
|
||||
hidden_dim,
|
||||
include_conv5=False,
|
||||
dropout_rate=0.1,
|
||||
positional_encoding_type="learned",
|
||||
backbone='r50x1',
|
||||
):
|
||||
super(ResNetHybridViT, self).__init__(
|
||||
img_dim=img_dim,
|
||||
out_dim=out_dim,
|
||||
num_channels=num_channels,
|
||||
embedding_dim=embedding_dim,
|
||||
num_heads=num_heads,
|
||||
num_layers=num_layers,
|
||||
hidden_dim=hidden_dim,
|
||||
include_conv5=include_conv5,
|
||||
dropout_rate=dropout_rate,
|
||||
positional_encoding_type=positional_encoding_type,
|
||||
backbone=backbone,
|
||||
)
|
||||
|
||||
def configure_backbone(self):
|
||||
"""
|
||||
Current support offered for all BiT models
|
||||
KNOWN_MODELS in https://github.com/google-research/big_transfer/blob/master/bit_pytorch/models.py
|
||||
|
||||
expects model name of style 'r{depth}x{width}'
|
||||
where depth in [50, 101, 152]
|
||||
where width in [1,2,3,4]
|
||||
"""
|
||||
backbone = self.backbone
|
||||
out_dim = self.out_dim
|
||||
|
||||
splits = backbone.split('x')
|
||||
model_name = splits[0]
|
||||
width_factor = int(splits[1])
|
||||
|
||||
if model_name in ['r50', 'r101'] and width_factor in [2, 4]:
|
||||
return ValueError(
|
||||
"Invalid Configuration of models -- expect 50x1, 50x3, 101x1, 101x3"
|
||||
)
|
||||
elif model_name == 'r152' and width_factor in [1, 3]:
|
||||
return ValueError(
|
||||
"Invalid Configuration of models -- expect 152x2, 152x4"
|
||||
)
|
||||
|
||||
block_units_dict = {
|
||||
'r50': [3, 4, 6, 3],
|
||||
'r101': [3, 4, 23, 3],
|
||||
'r152': [3, 8, 36, 3],
|
||||
}
|
||||
block_units = block_units_dict.get(model_name, [3, 4, 6, 3])
|
||||
model = ResNetV2Model(block_units, width_factor, head_size=out_dim)
|
||||
|
||||
if self.num_channels == 3:
|
||||
flatten_dim = 1024 * width_factor
|
||||
if self.include_conv5:
|
||||
flatten_dim *= 2
|
||||
|
||||
return model, flatten_dim
|
||||
|
||||
|
||||
class AxialNetHybridViT(HybridVisionTransformer):
|
||||
def __init__(
|
||||
self,
|
||||
img_dim,
|
||||
out_dim,
|
||||
num_channels,
|
||||
embedding_dim,
|
||||
num_heads,
|
||||
num_layers,
|
||||
hidden_dim,
|
||||
include_conv5=False,
|
||||
dropout_rate=0.1,
|
||||
positional_encoding_type="learned",
|
||||
backbone='a50m',
|
||||
):
|
||||
super(AxialNetHybridViT, self).__init__(
|
||||
img_dim=img_dim,
|
||||
out_dim=out_dim,
|
||||
num_channels=num_channels,
|
||||
embedding_dim=embedding_dim,
|
||||
num_heads=num_heads,
|
||||
num_layers=num_layers,
|
||||
hidden_dim=hidden_dim,
|
||||
include_conv5=include_conv5,
|
||||
dropout_rate=dropout_rate,
|
||||
positional_encoding_type=positional_encoding_type,
|
||||
backbone=backbone,
|
||||
)
|
||||
|
||||
def configure_backbone(self):
|
||||
"""
|
||||
Current support offered for all BiT models
|
||||
models from https://github.com/csrhddlam/axial-deeplab/blob/master/lib/models/axialnet.py
|
||||
|
||||
expects model name of style 'a{depth}{width}'
|
||||
where depth in [26, 50, 101]
|
||||
where width in [s, m, l]
|
||||
"""
|
||||
backbone = self.backbone
|
||||
out_dim = self.out_dim
|
||||
|
||||
model_name = backbone[:3]
|
||||
width = backbone[-1]
|
||||
|
||||
block_units_dict = {
|
||||
'a26': [1, 2, 4, 1],
|
||||
'a50': [3, 4, 6, 3],
|
||||
'a101': [3, 4, 23, 3],
|
||||
}
|
||||
block_units = block_units_dict.get(model_name, [3, 4, 6, 3])
|
||||
|
||||
scale_factor_dict = {'s': 0.5, 'm': 0.75, 'l': 1.0}
|
||||
scale_factor = scale_factor_dict.get(width, 0.75)
|
||||
model = AxialAttentionNet(
|
||||
block_units, s=scale_factor, num_classes=out_dim
|
||||
)
|
||||
|
||||
if self.num_channels == 3:
|
||||
flatten_dim = int(512 * float(scale_factor / 0.5))
|
||||
if self.include_conv5:
|
||||
flatten_dim *= 2
|
||||
|
||||
return model, flatten_dim
|
||||
331
third_party/EgoSpeak/MiniROAD/model/transformer_models/OADTR_ViT_origin.py
vendored
Normal file
331
third_party/EgoSpeak/MiniROAD/model/transformer_models/OADTR_ViT_origin.py
vendored
Normal file
@@ -0,0 +1,331 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from .decoder import Decoder, DecoderLayer
|
||||
from .attn import FullAttention, ProbAttention, AttentionLayer
|
||||
from .Transformer import TransformerModel
|
||||
from ipdb import set_trace
|
||||
from .PositionalEncoding import (
|
||||
FixedPositionalEncoding,
|
||||
LearnedPositionalEncoding,
|
||||
)
|
||||
|
||||
__all__ = ['ViT_B16', 'ViT_B32', 'ViT_L16', 'ViT_L32', 'ViT_H14']
|
||||
|
||||
|
||||
class VisionTransformer_v3(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
args,
|
||||
img_dim,
|
||||
patch_dim,
|
||||
out_dim,
|
||||
embedding_dim,
|
||||
num_heads,
|
||||
num_layers,
|
||||
hidden_dim,
|
||||
dropout_rate=0.0,
|
||||
attn_dropout_rate=0.0,
|
||||
use_representation=True,
|
||||
conv_patch_representation=False,
|
||||
positional_encoding_type="learned", with_camera=True, with_motion=True, num_channels=3072,
|
||||
):
|
||||
super(VisionTransformer_v3, self).__init__()
|
||||
|
||||
assert embedding_dim % num_heads == 0
|
||||
assert img_dim % patch_dim == 0
|
||||
self.with_camera = with_camera
|
||||
self.with_motion = with_motion
|
||||
self.embedding_dim = embedding_dim
|
||||
self.num_heads = num_heads
|
||||
self.patch_dim = patch_dim
|
||||
# num_channels = img_dim
|
||||
self.num_channels = num_channels
|
||||
self.dropout_rate = dropout_rate
|
||||
self.attn_dropout_rate = attn_dropout_rate
|
||||
self.conv_patch_representation = conv_patch_representation
|
||||
|
||||
# self.num_patches = int((img_dim // patch_dim) ** 2)
|
||||
self.num_patches = int(img_dim // patch_dim)
|
||||
self.seq_length = self.num_patches + 1
|
||||
self.flatten_dim = patch_dim * patch_dim * num_channels
|
||||
self.cls_token = nn.Parameter(torch.zeros(1, 1, embedding_dim))
|
||||
|
||||
self.linear_encoding = nn.Linear(self.flatten_dim, embedding_dim)
|
||||
if positional_encoding_type == "learned":
|
||||
self.position_encoding = LearnedPositionalEncoding(
|
||||
self.seq_length, self.embedding_dim, self.seq_length
|
||||
)
|
||||
elif positional_encoding_type == "fixed":
|
||||
self.position_encoding = FixedPositionalEncoding(
|
||||
self.embedding_dim,
|
||||
)
|
||||
print('position encoding :', positional_encoding_type)
|
||||
|
||||
self.pe_dropout = nn.Dropout(p=self.dropout_rate)
|
||||
|
||||
self.encoder = TransformerModel(
|
||||
embedding_dim,
|
||||
num_layers,
|
||||
num_heads,
|
||||
hidden_dim,
|
||||
self.dropout_rate,
|
||||
self.attn_dropout_rate,
|
||||
)
|
||||
self.pre_head_ln = nn.LayerNorm(embedding_dim)
|
||||
|
||||
d_model = args.decoder_embedding_dim
|
||||
use_representation = False # False
|
||||
if use_representation:
|
||||
self.mlp_head = nn.Sequential(
|
||||
nn.Linear(embedding_dim + d_model, hidden_dim//2),
|
||||
# nn.Tanh(),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim//2, out_dim),
|
||||
)
|
||||
else:
|
||||
self.mlp_head = nn.Linear(embedding_dim + d_model, out_dim)
|
||||
|
||||
if self.conv_patch_representation:
|
||||
# self.conv_x = nn.Conv2d(
|
||||
# self.num_channels,
|
||||
# self.embedding_dim,
|
||||
# kernel_size=(self.patch_dim, self.patch_dim),
|
||||
# stride=(self.patch_dim, self.patch_dim),
|
||||
# padding=self._get_padding(
|
||||
# 'VALID', (self.patch_dim, self.patch_dim),
|
||||
# ),
|
||||
# )
|
||||
self.conv_x = nn.Conv1d(
|
||||
self.num_channels,
|
||||
self.embedding_dim,
|
||||
kernel_size=self.patch_dim,
|
||||
stride=self.patch_dim,
|
||||
padding=self._get_padding(
|
||||
'VALID', (self.patch_dim),
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.conv_x = None
|
||||
|
||||
self.to_cls_token = nn.Identity()
|
||||
|
||||
# Decoder
|
||||
factor = 1 # 5
|
||||
dropout = args.decoder_attn_dropout_rate
|
||||
# d_model = args.decoder_embedding_dim
|
||||
n_heads = args.decoder_num_heads
|
||||
d_layers = args.decoder_layers
|
||||
d_ff = args.decoder_embedding_dim_out # args.decoder_embedding_dim_out or 4*args.decoder_embedding_dim None
|
||||
activation = 'gelu' # 'gelu'
|
||||
self.decoder = Decoder(
|
||||
[
|
||||
DecoderLayer(
|
||||
AttentionLayer(FullAttention(True, factor, attention_dropout=dropout), # True
|
||||
d_model, n_heads), # ProbAttention FullAttention
|
||||
AttentionLayer(FullAttention(False, factor, attention_dropout=dropout), # False
|
||||
d_model, n_heads),
|
||||
d_model,
|
||||
d_ff,
|
||||
dropout=dropout,
|
||||
activation=activation,
|
||||
)
|
||||
for l in range(d_layers)
|
||||
],
|
||||
norm_layer=torch.nn.LayerNorm(d_model)
|
||||
)
|
||||
self.decoder_cls_token = nn.Parameter(torch.zeros(1, args.query_num, d_model))
|
||||
if positional_encoding_type == "learned":
|
||||
self.decoder_position_encoding = LearnedPositionalEncoding(
|
||||
args.query_num, self.embedding_dim, args.query_num
|
||||
)
|
||||
elif positional_encoding_type == "fixed":
|
||||
self.decoder_position_encoding = FixedPositionalEncoding(
|
||||
self.embedding_dim,
|
||||
)
|
||||
print('position decoding :', positional_encoding_type)
|
||||
self.classifier = nn.Linear(d_model, out_dim)
|
||||
self.after_dropout = nn.Dropout(p=self.dropout_rate)
|
||||
# self.merge_fc = nn.Linear(d_model, 1)
|
||||
# self.merge_sigmoid = nn.Sigmoid()
|
||||
|
||||
def forward(self, sequence_input_rgb, sequence_input_flow):
|
||||
if self.with_camera and self.with_motion:
|
||||
x = torch.cat((sequence_input_rgb, sequence_input_flow), 2)
|
||||
elif self.with_camera:
|
||||
x = sequence_input_rgb
|
||||
elif self.with_motion:
|
||||
x = sequence_input_flow
|
||||
|
||||
x = self.linear_encoding(x)
|
||||
cls_tokens = self.cls_token.expand(x.shape[0], -1, -1)
|
||||
# x = torch.cat((cls_tokens, x), dim=1)
|
||||
x = torch.cat((x, cls_tokens), dim=1)
|
||||
x = self.position_encoding(x)
|
||||
x = self.pe_dropout(x) # not delete
|
||||
|
||||
# apply transformer
|
||||
x = self.encoder(x)
|
||||
x = self.pre_head_ln(x) # [128, 33, 1024]
|
||||
# x = self.after_dropout(x) # add
|
||||
# decoder
|
||||
decoder_cls_token = self.decoder_cls_token.expand(x.shape[0], -1, -1)
|
||||
# decoder_cls_token = self.after_dropout(decoder_cls_token) # add
|
||||
# decoder_cls_token = self.decoder_position_encoding(decoder_cls_token) # [128, 8, 1024]
|
||||
dec = self.decoder(decoder_cls_token, x) # [128, 8, 1024]
|
||||
dec = self.after_dropout(dec) # add
|
||||
# merge_atte = self.merge_sigmoid(self.merge_fc(dec)) # [128, 8, 1]
|
||||
# dec_for_token = (merge_atte*dec).sum(dim=1) # [128, 1024]
|
||||
# dec_for_token = (merge_atte*dec).sum(dim=1)/(merge_atte.sum(dim=-2) + 0.0001)
|
||||
dec_for_token = dec.mean(dim=1)
|
||||
# dec_for_token = dec.max(dim=1)[0]
|
||||
dec_cls_out = self.classifier(dec)
|
||||
# set_trace()
|
||||
# x = self.to_cls_token(x[:, 0])
|
||||
x = torch.cat((self.to_cls_token(x[:, -1]), dec_for_token), dim=1)
|
||||
x = self.mlp_head(x)
|
||||
# x = F.log_softmax(x, dim=-1)
|
||||
|
||||
return x, dec_cls_out
|
||||
|
||||
def _get_padding(self, padding_type, kernel_size):
|
||||
assert padding_type in ['SAME', 'VALID']
|
||||
if padding_type == 'SAME':
|
||||
_list = [(k - 1) // 2 for k in kernel_size]
|
||||
return tuple(_list)
|
||||
return tuple(0 for _ in kernel_size)
|
||||
|
||||
|
||||
def ViT_B16(dataset='imagenet'):
|
||||
if dataset == 'imagenet':
|
||||
img_dim = 224
|
||||
out_dim = 1000
|
||||
patch_dim = 16
|
||||
elif 'cifar' in dataset:
|
||||
img_dim = 32
|
||||
out_dim = 10
|
||||
patch_dim = 4
|
||||
|
||||
return VisionTransformer(
|
||||
img_dim=img_dim,
|
||||
patch_dim=patch_dim,
|
||||
out_dim=out_dim,
|
||||
num_channels=3,
|
||||
embedding_dim=768,
|
||||
num_heads=12,
|
||||
num_layers=12,
|
||||
hidden_dim=3072,
|
||||
dropout_rate=0.1,
|
||||
attn_dropout_rate=0.0,
|
||||
use_representation=False,
|
||||
conv_patch_representation=False,
|
||||
positional_encoding_type="learned",
|
||||
)
|
||||
|
||||
|
||||
def ViT_B32(dataset='imagenet'):
|
||||
if dataset == 'imagenet':
|
||||
img_dim = 224
|
||||
out_dim = 1000
|
||||
patch_dim = 32
|
||||
elif 'cifar' in dataset:
|
||||
img_dim = 32
|
||||
out_dim = 10
|
||||
patch_dim = 4
|
||||
|
||||
return VisionTransformer(
|
||||
img_dim=img_dim,
|
||||
patch_dim=patch_dim,
|
||||
out_dim=out_dim,
|
||||
num_channels=3,
|
||||
embedding_dim=768,
|
||||
num_heads=12,
|
||||
num_layers=12,
|
||||
hidden_dim=3072,
|
||||
dropout_rate=0.1,
|
||||
attn_dropout_rate=0.0,
|
||||
use_representation=False,
|
||||
conv_patch_representation=False,
|
||||
positional_encoding_type="learned",
|
||||
)
|
||||
|
||||
|
||||
def ViT_L16(dataset='imagenet'):
|
||||
if dataset == 'imagenet':
|
||||
img_dim = 224
|
||||
out_dim = 1000
|
||||
patch_dim = 16
|
||||
elif 'cifar' in dataset:
|
||||
img_dim = 32
|
||||
out_dim = 10
|
||||
patch_dim = 4
|
||||
|
||||
return VisionTransformer(
|
||||
img_dim=img_dim,
|
||||
patch_dim=patch_dim,
|
||||
out_dim=out_dim,
|
||||
num_channels=3,
|
||||
embedding_dim=1024,
|
||||
num_heads=16,
|
||||
num_layers=24,
|
||||
hidden_dim=4096,
|
||||
dropout_rate=0.1,
|
||||
attn_dropout_rate=0.0,
|
||||
use_representation=False,
|
||||
conv_patch_representation=False,
|
||||
positional_encoding_type="learned",
|
||||
)
|
||||
|
||||
|
||||
def ViT_L32(dataset='imagenet'):
|
||||
if dataset == 'imagenet':
|
||||
img_dim = 224
|
||||
out_dim = 1000
|
||||
patch_dim = 32
|
||||
elif 'cifar' in dataset:
|
||||
img_dim = 32
|
||||
out_dim = 10
|
||||
patch_dim = 4
|
||||
|
||||
return VisionTransformer(
|
||||
img_dim=img_dim,
|
||||
patch_dim=patch_dim,
|
||||
out_dim=out_dim,
|
||||
num_channels=3,
|
||||
embedding_dim=1024,
|
||||
num_heads=16,
|
||||
num_layers=24,
|
||||
hidden_dim=4096,
|
||||
dropout_rate=0.1,
|
||||
attn_dropout_rate=0.0,
|
||||
use_representation=False,
|
||||
conv_patch_representation=False,
|
||||
positional_encoding_type="learned",
|
||||
)
|
||||
|
||||
|
||||
def ViT_H14(dataset='imagenet'):
|
||||
if dataset == 'imagenet':
|
||||
img_dim = 224
|
||||
out_dim = 1000
|
||||
patch_dim = 14
|
||||
elif 'cifar' in dataset:
|
||||
img_dim = 32
|
||||
out_dim = 10
|
||||
patch_dim = 4
|
||||
|
||||
return VisionTransformer(
|
||||
img_dim=img_dim,
|
||||
patch_dim=patch_dim,
|
||||
out_dim=out_dim,
|
||||
num_channels=3,
|
||||
embedding_dim=1280,
|
||||
num_heads=16,
|
||||
num_layers=32,
|
||||
hidden_dim=5120,
|
||||
dropout_rate=0.1,
|
||||
attn_dropout_rate=0.0,
|
||||
use_representation=False,
|
||||
conv_patch_representation=False,
|
||||
positional_encoding_type="learned",
|
||||
)
|
||||
44
third_party/EgoSpeak/MiniROAD/model/transformer_models/PositionalEncoding.py
vendored
Executable file
44
third_party/EgoSpeak/MiniROAD/model/transformer_models/PositionalEncoding.py
vendored
Executable file
@@ -0,0 +1,44 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
class FixedPositionalEncoding(nn.Module):
|
||||
def __init__(self, embedding_dim, max_length=5000):
|
||||
super(FixedPositionalEncoding, self).__init__()
|
||||
|
||||
pe = torch.zeros(max_length, embedding_dim)
|
||||
position = torch.arange(0, max_length, dtype=torch.float).unsqueeze(1)
|
||||
div_term = torch.exp(
|
||||
torch.arange(0, embedding_dim, 2).float()
|
||||
* (-torch.log(torch.tensor(10000.0)) / embedding_dim)
|
||||
)
|
||||
pe[:, 0::2] = torch.sin(position * div_term)
|
||||
pe[:, 1::2] = torch.cos(position * div_term)
|
||||
pe = pe.unsqueeze(0).transpose(0, 1)
|
||||
self.register_buffer('pe', pe)
|
||||
|
||||
def forward(self, x):
|
||||
x = x + self.pe[: x.size(0), :]
|
||||
return x
|
||||
|
||||
|
||||
class LearnedPositionalEncoding(nn.Module):
|
||||
def __init__(self, max_position_embeddings, embedding_dim, seq_length):
|
||||
super(LearnedPositionalEncoding, self).__init__()
|
||||
self.pe = nn.Embedding(max_position_embeddings, embedding_dim)
|
||||
self.seq_length = seq_length
|
||||
|
||||
self.register_buffer(
|
||||
"position_ids",
|
||||
torch.arange(max_position_embeddings).expand((1, -1)),
|
||||
)
|
||||
|
||||
def forward(self, x, position_ids=None):
|
||||
if position_ids is None:
|
||||
position_ids = self.position_ids[:, : self.seq_length]
|
||||
|
||||
try:
|
||||
position_embeddings = self.pe(position_ids)
|
||||
return x + position_embeddings
|
||||
except:
|
||||
import pdb; pdb.set_trace()
|
||||
82
third_party/EgoSpeak/MiniROAD/model/transformer_models/Transformer.py
vendored
Executable file
82
third_party/EgoSpeak/MiniROAD/model/transformer_models/Transformer.py
vendored
Executable file
@@ -0,0 +1,82 @@
|
||||
from torch import nn
|
||||
from .Attention import SelfAttention
|
||||
|
||||
|
||||
class Residual(nn.Module):
|
||||
def __init__(self, fn):
|
||||
super().__init__()
|
||||
self.fn = fn
|
||||
|
||||
def forward(self, x):
|
||||
return self.fn(x) + x
|
||||
|
||||
|
||||
class PreNorm(nn.Module):
|
||||
def __init__(self, dim, fn):
|
||||
super().__init__()
|
||||
self.norm = nn.LayerNorm(dim)
|
||||
self.fn = fn
|
||||
|
||||
def forward(self, x):
|
||||
return self.fn(self.norm(x))
|
||||
|
||||
|
||||
class PreNormDrop(nn.Module):
|
||||
def __init__(self, dim, dropout_rate, fn):
|
||||
super().__init__()
|
||||
self.norm = nn.LayerNorm(dim)
|
||||
self.dropout = nn.Dropout(p=dropout_rate)
|
||||
self.fn = fn
|
||||
|
||||
def forward(self, x):
|
||||
return self.dropout(self.fn(self.norm(x)))
|
||||
|
||||
|
||||
class FeedForward(nn.Module):
|
||||
def __init__(self, dim, hidden_dim, dropout_rate):
|
||||
super().__init__()
|
||||
self.net = nn.Sequential(
|
||||
nn.Linear(dim, hidden_dim),
|
||||
nn.GELU(),
|
||||
nn.Dropout(p=dropout_rate),
|
||||
nn.Linear(hidden_dim, dim),
|
||||
nn.Dropout(p=dropout_rate),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
|
||||
class TransformerModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
depth,
|
||||
heads,
|
||||
mlp_dim,
|
||||
dropout_rate=0.1,
|
||||
attn_dropout_rate=0.1,
|
||||
):
|
||||
super().__init__()
|
||||
layers = []
|
||||
for _ in range(depth):
|
||||
layers.extend(
|
||||
[
|
||||
Residual(
|
||||
PreNormDrop(
|
||||
dim,
|
||||
dropout_rate,
|
||||
SelfAttention(
|
||||
dim, heads=heads, dropout_rate=attn_dropout_rate
|
||||
),
|
||||
)
|
||||
),
|
||||
Residual(
|
||||
PreNorm(dim, FeedForward(dim, mlp_dim, dropout_rate))
|
||||
),
|
||||
]
|
||||
)
|
||||
self.net = nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
350
third_party/EgoSpeak/MiniROAD/model/transformer_models/ViT.py
vendored
Executable file
350
third_party/EgoSpeak/MiniROAD/model/transformer_models/ViT.py
vendored
Executable file
@@ -0,0 +1,350 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from configs.feature_sizes import FEATURE_SIZES
|
||||
from model.model_builder import META_ARCHITECTURES as registry
|
||||
|
||||
from .attn import AttentionLayer, FullAttention, ProbAttention
|
||||
from .decoder import Decoder, DecoderLayer
|
||||
from .PositionalEncoding import (FixedPositionalEncoding,
|
||||
LearnedPositionalEncoding)
|
||||
from .Transformer import TransformerModel
|
||||
|
||||
|
||||
# @registry.register('TRANSFORMER')
|
||||
@registry.register('OADTR')
|
||||
class OADTR(nn.Module):
|
||||
def __init__(self, cfg, use_representation=True,
|
||||
conv_patch_representation=False,
|
||||
positional_encoding_type="learned"
|
||||
):
|
||||
super(OADTR, self).__init__()
|
||||
|
||||
self.img_dim = cfg["window_size"] # img_dim
|
||||
self.out_dim = cfg["num_classes"]
|
||||
|
||||
self.patch_dim = cfg["patch_dim"]
|
||||
self.query_num = cfg["query_num"]
|
||||
|
||||
self.embedding_dim = cfg["embedding_dim"]
|
||||
self.num_heads = cfg["num_heads"]
|
||||
self.num_layers = cfg["num_layers"]
|
||||
self.hidden_dim = cfg["hidden_dim"]
|
||||
|
||||
self.dropout_rate = cfg["dropout"]
|
||||
self.attn_dropout_rate = cfg["attn_dropout_rate"]
|
||||
|
||||
self.conv_patch_representation = conv_patch_representation
|
||||
|
||||
self.decoder_embedding_dim = cfg["decoder_embedding_dim"]
|
||||
self.decoder_num_heads = cfg["decoder_num_heads"]
|
||||
self.decoder_layers = cfg["decoder_layers"]
|
||||
self.decoder_embedding_dim_out = cfg["decoder_embedding_dim_out"]
|
||||
self.decoder_attn_dropout_rate = cfg["decoder_attn_dropout_rate"]
|
||||
|
||||
self.use_flow = not cfg['no_flow']
|
||||
self.use_rgb = not cfg['no_rgb']
|
||||
|
||||
self.num_channels = 0
|
||||
if self.use_rgb:
|
||||
self.num_channels += FEATURE_SIZES[cfg['rgb_type']]
|
||||
if self.use_flow:
|
||||
self.num_channels += FEATURE_SIZES[cfg['flow_type']]
|
||||
|
||||
assert self.embedding_dim % self.num_heads == 0
|
||||
assert self.img_dim % self.patch_dim == 0
|
||||
|
||||
self.num_patches = int(self.img_dim // self.patch_dim)
|
||||
self.seq_length = self.num_patches + 1
|
||||
# self.seq_length = self.num_patches + self.img_dim
|
||||
self.flatten_dim = self.patch_dim * self.patch_dim * self.num_channels
|
||||
self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embedding_dim))
|
||||
# self.cls_token = nn.Parameter(torch.zeros(1, self.img_dim, self.embedding_dim))
|
||||
|
||||
self.linear_encoding = nn.Linear(self.flatten_dim, self.embedding_dim)
|
||||
if positional_encoding_type == "learned":
|
||||
self.position_encoding = LearnedPositionalEncoding(
|
||||
self.seq_length, self.embedding_dim, self.seq_length
|
||||
)
|
||||
elif positional_encoding_type == "fixed":
|
||||
self.position_encoding = FixedPositionalEncoding(
|
||||
self.embedding_dim,
|
||||
)
|
||||
print('position encoding :', positional_encoding_type)
|
||||
|
||||
self.pe_dropout = nn.Dropout(p=self.dropout_rate)
|
||||
|
||||
self.encoder = TransformerModel(
|
||||
self.embedding_dim,
|
||||
self.num_layers,
|
||||
self.num_heads,
|
||||
self.hidden_dim,
|
||||
self.dropout_rate,
|
||||
self.attn_dropout_rate,
|
||||
)
|
||||
self.pre_head_ln = nn.LayerNorm(self.embedding_dim)
|
||||
|
||||
d_model = self.decoder_embedding_dim
|
||||
use_representation = False # False
|
||||
if use_representation:
|
||||
self.mlp_head = nn.Sequential(
|
||||
nn.Linear(self.embedding_dim , self.hidden_dim//2),
|
||||
# nn.Tanh(),
|
||||
nn.ReLU(),
|
||||
nn.Linear(self.hidden_dim//2, self.out_dim),
|
||||
)
|
||||
else:
|
||||
self.mlp_head = nn.Linear(self.embedding_dim + d_model, self.out_dim)
|
||||
|
||||
if self.conv_patch_representation:
|
||||
# self.conv_x = nn.Conv2d(
|
||||
# self.num_channels,
|
||||
# self.embedding_dim,
|
||||
# kernel_size=(self.patch_dim, self.patch_dim),
|
||||
# stride=(self.patch_dim, self.patch_dim),
|
||||
# padding=self._get_padding(
|
||||
# 'VALID', (self.patch_dim, self.patch_dim),
|
||||
# ),
|
||||
# )
|
||||
self.conv_x = nn.Conv1d(
|
||||
self.num_channels,
|
||||
self.embedding_dim,
|
||||
kernel_size=self.patch_dim,
|
||||
stride=self.patch_dim,
|
||||
padding=self._get_padding(
|
||||
'VALID', (self.patch_dim),
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.conv_x = None
|
||||
|
||||
self.to_cls_token = nn.Identity()
|
||||
|
||||
# Decoder
|
||||
factor = 1 # 5
|
||||
dropout = self.decoder_attn_dropout_rate
|
||||
# d_model = self.decoder_embedding_dim
|
||||
n_heads = self.decoder_num_heads
|
||||
d_layers = self.decoder_layers
|
||||
d_ff = self.decoder_embedding_dim_out # args.decoder_embedding_dim_out or 4*args.decoder_embedding_dim None
|
||||
activation = 'gelu' # 'gelu'
|
||||
self.decoder = Decoder(
|
||||
[
|
||||
DecoderLayer(
|
||||
AttentionLayer(FullAttention(True, factor, attention_dropout=dropout), # True
|
||||
d_model, n_heads), # ProbAttention FullAttention
|
||||
AttentionLayer(FullAttention(False, factor, attention_dropout=dropout), # False
|
||||
d_model, n_heads),
|
||||
d_model,
|
||||
d_ff,
|
||||
dropout=dropout,
|
||||
activation=activation,
|
||||
)
|
||||
for l in range(d_layers)
|
||||
],
|
||||
norm_layer=torch.nn.LayerNorm(d_model)
|
||||
)
|
||||
self.decoder_cls_token = nn.Parameter(torch.zeros(1, self.query_num, d_model))
|
||||
if positional_encoding_type == "learned":
|
||||
self.decoder_position_encoding = LearnedPositionalEncoding(
|
||||
self.query_num, self.embedding_dim, self.query_num
|
||||
)
|
||||
elif positional_encoding_type == "fixed":
|
||||
self.decoder_position_encoding = FixedPositionalEncoding(
|
||||
self.embedding_dim,
|
||||
)
|
||||
print('position decoding :', positional_encoding_type)
|
||||
self.classifier = nn.Linear(d_model, self.out_dim)
|
||||
self.after_dropout = nn.Dropout(p=self.dropout_rate)
|
||||
# self.merge_fc = nn.Linear(d_model, 1)
|
||||
# self.merge_sigmoid = nn.Sigmoid()
|
||||
|
||||
|
||||
def forward(self, sequence_input_rgb, sequence_input_flow):
|
||||
if self.use_rgb and self.use_flow:
|
||||
x = torch.cat((sequence_input_rgb, sequence_input_flow), 2)
|
||||
elif self.use_rgb:
|
||||
x = sequence_input_rgb
|
||||
elif self.use_flow:
|
||||
x = sequence_input_flow
|
||||
|
||||
x = self.linear_encoding(x) # B, 128, 1204
|
||||
cls_tokens = self.cls_token.expand(x.shape[0], -1, -1) # B, 1, 1024
|
||||
# x = torch.cat((cls_tokens, x), dim=1)
|
||||
x = torch.cat((x, cls_tokens), dim=1) # B, seq+1, 1024
|
||||
try:
|
||||
x = self.position_encoding(x) # B, seq+1, 1024
|
||||
except:
|
||||
import pdb; pdb.set_trace()
|
||||
x = self.pe_dropout(x) # not delete
|
||||
|
||||
# apply transformer
|
||||
x = self.encoder(x)
|
||||
x = self.pre_head_ln(x) # B, seq+1, 1024
|
||||
|
||||
# decoder
|
||||
decoder_cls_token = self.decoder_cls_token.expand(x.shape[0], -1, -1) # [B, 8, 1024]
|
||||
# decoder_cls_token = self.after_dropout(decoder_cls_token) # add
|
||||
# decoder_cls_token = self.decoder_position_encoding(decoder_cls_token) # [128, 8, 1024]
|
||||
dec = self.decoder(decoder_cls_token, x) # [128, 8, 1024]
|
||||
dec = self.after_dropout(dec) # add
|
||||
|
||||
# merge_atte = self.merge_sigmoid(self.merge_fc(dec)) # [128, 8, 1]
|
||||
# dec_for_token = (merge_atte*dec).sum(dim=1) # [128, 1024]
|
||||
# dec_for_token = (merge_atte*dec).sum(dim=1)/(merge_atte.sum(dim=-2) + 0.0001)
|
||||
# *** RuntimeError: mat1 and mat2 shapes cannot be multiplied (64x2048 and 1024x2)
|
||||
dec_for_token = dec.mean(dim=1) # [64, 1024]
|
||||
|
||||
# dec_for_token = dec.max(dim=1)[0]
|
||||
dec_cls_out = self.classifier(dec) # [B, 8, NUM_CLASS?]
|
||||
# set_trace()
|
||||
# x = self.to_cls_token(x[:, 0])
|
||||
|
||||
x = torch.cat((self.to_cls_token(x[:, -1]), dec_for_token), dim=1) # [B, 2048]
|
||||
x = self.mlp_head(x) # [B, NUM_CLASSES]
|
||||
|
||||
# original OADTR
|
||||
# return x, dec_cls_out
|
||||
|
||||
# x = F.log_softmax(x, dim=-1)
|
||||
out_dict = {}
|
||||
out_dict['logits'] = x.unsqueeze(1) # [B, 1, NUM_CLASSES] ex) [64, 1, 2]
|
||||
out_dict['dec_logits'] = dec_cls_out # [B, 8, NUM_CLASSES] ex) [64, 8, 2]
|
||||
|
||||
# out_dict['logits'] = x #x.unsqueeze(1)
|
||||
return out_dict
|
||||
|
||||
def _get_padding(self, padding_type, kernel_size):
|
||||
assert padding_type in ['SAME', 'VALID']
|
||||
if padding_type == 'SAME':
|
||||
_list = [(k - 1) // 2 for k in kernel_size]
|
||||
return tuple(_list)
|
||||
return tuple(0 for _ in kernel_size)
|
||||
|
||||
@registry.register('Transformer')
|
||||
class ViTEnc(nn.Module):
|
||||
def __init__(self, cfg, use_representation=True,
|
||||
conv_patch_representation=False,
|
||||
positional_encoding_type="learned"
|
||||
):
|
||||
super(ViTEnc, self).__init__()
|
||||
self.img_dim = cfg["window_size"]
|
||||
self.out_dim = cfg["num_classes"]
|
||||
self.embedding_dim = cfg["embedding_dim"]
|
||||
self.patch_dim = cfg["patch_dim"]
|
||||
self.num_heads = cfg["num_heads"]
|
||||
self.num_layers = cfg["num_layers"]
|
||||
self.hidden_dim = cfg["hidden_dim"]
|
||||
self.dropout_rate = cfg["dropout"]
|
||||
self.use_flow = not cfg['no_flow']
|
||||
self.use_rgb = not cfg['no_rgb']
|
||||
self.num_channels= 0
|
||||
if self.use_rgb:
|
||||
self.num_channels += FEATURE_SIZES[cfg['rgb_type']]
|
||||
if self.use_flow:
|
||||
self.num_channels += FEATURE_SIZES[cfg['flow_type']]
|
||||
self.attn_dropout_rate = cfg["attn_dropout_rate"]
|
||||
assert self.embedding_dim % self.num_heads == 0
|
||||
assert self.img_dim % self.patch_dim == 0
|
||||
self.conv_patch_representation = conv_patch_representation
|
||||
self.num_patches = int(self.img_dim // self.patch_dim)
|
||||
self.seq_length = self.num_patches + 1
|
||||
# self.seq_length = self.num_patches + self.img_dim
|
||||
self.flatten_dim = self.patch_dim * self.patch_dim * self.num_channels
|
||||
self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embedding_dim))
|
||||
# self.cls_token = nn.Parameter(torch.zeros(1, self.img_dim, self.embedding_dim))
|
||||
|
||||
self.linear_encoding = nn.Linear(self.flatten_dim, self.embedding_dim)
|
||||
if positional_encoding_type == "learned":
|
||||
self.position_encoding = LearnedPositionalEncoding(
|
||||
self.seq_length, self.embedding_dim, self.seq_length
|
||||
)
|
||||
elif positional_encoding_type == "fixed":
|
||||
self.position_encoding = FixedPositionalEncoding(
|
||||
self.embedding_dim,
|
||||
)
|
||||
print('position encoding :', positional_encoding_type)
|
||||
|
||||
self.pe_dropout = nn.Dropout(p=self.dropout_rate)
|
||||
|
||||
self.encoder = TransformerModel(
|
||||
self.embedding_dim,
|
||||
self.num_layers,
|
||||
self.num_heads,
|
||||
self.hidden_dim,
|
||||
self.dropout_rate,
|
||||
self.attn_dropout_rate,
|
||||
)
|
||||
self.pre_head_ln = nn.LayerNorm(self.embedding_dim)
|
||||
|
||||
use_representation = False # False
|
||||
if use_representation:
|
||||
self.mlp_head = nn.Sequential(
|
||||
nn.Linear(self.embedding_dim , self.hidden_dim//2),
|
||||
# nn.Tanh(),
|
||||
nn.ReLU(),
|
||||
nn.Linear(self.hidden_dim//2, self.out_dim),
|
||||
)
|
||||
else:
|
||||
self.mlp_head = nn.Linear(self.embedding_dim, self.out_dim)
|
||||
|
||||
if self.conv_patch_representation:
|
||||
# self.conv_x = nn.Conv2d(
|
||||
# self.num_channels,
|
||||
# self.embedding_dim,
|
||||
# kernel_size=(self.patch_dim, self.patch_dim),
|
||||
# stride=(self.patch_dim, self.patch_dim),
|
||||
# padding=self._get_padding(
|
||||
# 'VALID', (self.patch_dim, self.patch_dim),
|
||||
# ),
|
||||
# )
|
||||
self.conv_x = nn.Conv1d(
|
||||
self.num_channels,
|
||||
self.embedding_dim,
|
||||
kernel_size=self.patch_dim,
|
||||
stride=self.patch_dim,
|
||||
padding=self._get_padding(
|
||||
'VALID', (self.patch_dim),
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.conv_x = None
|
||||
|
||||
self.to_cls_token = nn.Identity()
|
||||
|
||||
|
||||
def forward(self, sequence_input_rgb, sequence_input_flow):
|
||||
if self.use_rgb and self.use_flow:
|
||||
x = torch.cat((sequence_input_rgb, sequence_input_flow), 2)
|
||||
elif self.use_rgb:
|
||||
x = sequence_input_rgb
|
||||
elif self.use_flow:
|
||||
x = sequence_input_flow
|
||||
|
||||
x = self.linear_encoding(x)
|
||||
cls_tokens = self.cls_token.expand(x.shape[0], -1, -1) # B, 1, 1024
|
||||
# x = torch.cat((cls_tokens, x), dim=1)
|
||||
x = torch.cat((x, cls_tokens), dim=1) # B, seq+1, 1024
|
||||
|
||||
x = self.position_encoding(x) # B, seq+1, 1024
|
||||
x = self.pe_dropout(x) # not delete
|
||||
|
||||
# apply transformer
|
||||
x = self.encoder(x)
|
||||
x = self.pre_head_ln(x) # B, seq+1, 1024
|
||||
|
||||
x = self.to_cls_token(x[:, 0]) # B, 1024
|
||||
# x = self.to_cls_token(x[:,0:self.img_dim]) # B, 1024
|
||||
x = self.mlp_head(x)
|
||||
# x = F.log_softmax(x, dim=-1)
|
||||
out_dict = {}
|
||||
out_dict['logits'] = x.unsqueeze(1)
|
||||
# out_dict['logits'] = x #x.unsqueeze(1)
|
||||
return out_dict
|
||||
|
||||
def _get_padding(self, padding_type, kernel_size):
|
||||
assert padding_type in ['SAME', 'VALID']
|
||||
if padding_type == 'SAME':
|
||||
_list = [(k - 1) // 2 for k in kernel_size]
|
||||
return tuple(_list)
|
||||
return tuple(0 for _ in kernel_size)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user