diff --git a/examples/eqv2-omat.ipynb b/examples/eqv2-omat.ipynb new file mode 100644 index 0000000..a86eb86 --- /dev/null +++ b/examples/eqv2-omat.ipynb @@ -0,0 +1,1345 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:matplotlib:matplotlib data path: /opt/conda/envs/jmp-peft/lib/python3.11/site-packages/matplotlib/mpl-data\n", + "DEBUG:matplotlib:CONFIGDIR=/root/.config/matplotlib\n", + "DEBUG:matplotlib:interactive is False\n", + "DEBUG:matplotlib:platform is linux\n", + "DEBUG:matplotlib:CONFIGDIR=/root/.config/matplotlib\n", + "DEBUG:matplotlib:interactive is False\n", + "DEBUG:matplotlib:platform is linux\n", + "DEBUG:matplotlib:CACHEDIR=/root/.cache/matplotlib\n", + "DEBUG:matplotlib.font_manager:Using fontManager instance from /root/.cache/matplotlib/fontlist-v390.json\n" + ] + } + ], + "source": [ + "import logging\n", + "\n", + "import nshutils as nu\n", + "import rich\n", + "\n", + "logging.basicConfig(level=logging.DEBUG)\n", + "\n", + "nu.pretty()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:nshconfig._src.registry:Registered with tag 'omat24'.\n", + "INFO:nshconfig._src.registry:Registered with tag 'xyz'.\n", + "INFO:nshconfig._src.registry:Registered with tag 'jmp'.\n", + "INFO:nshconfig._src.registry:Registered with tag 'm3gnet'.\n", + "INFO:nshconfig._src.registry:Registered with tag 'eqV2'.\n", + "INFO:nshconfig._src.registry:Registered with tag 'orb'.\n", + "INFO:nshconfig._src.registry:Registered with tag 'db'.\n", + "INFO:nshconfig._src.registry:Registered with tag 'json'.\n", + "INFO:nshconfig._src.registry:Registered with tag 'matbench'.\n" + ] + }, + { + "data": { + "text/html": [ + "
MatterTunerConfig(\n",
+       "    data=PerSplitDataConfig(\n",
+       "        train=OMAT24DatasetConfig(type='omat24', src=Path('/mnt/datasets/salex/val')),\n",
+       "        validation=None,\n",
+       "        batch_size=1,\n",
+       "        num_workers=0,\n",
+       "        pin_memory=True\n",
+       "    ),\n",
+       "    model=EqV2BackboneConfig(\n",
+       "        properties=[\n",
+       "            EnergyPropertyConfig(\n",
+       "                name='energy',\n",
+       "                dtype='float',\n",
+       "                loss=MAELossConfig(name='mae', reduction='mean'),\n",
+       "                loss_coefficient=1.0,\n",
+       "                type='energy'\n",
+       "            ),\n",
+       "            ForcesPropertyConfig(\n",
+       "                name='forces',\n",
+       "                dtype='float',\n",
+       "                loss=MAELossConfig(name='mae', reduction='mean'),\n",
+       "                loss_coefficient=1.0,\n",
+       "                type='forces',\n",
+       "                conservative=False\n",
+       "            ),\n",
+       "            StressesPropertyConfig(\n",
+       "                name='stresses',\n",
+       "                dtype='float',\n",
+       "                loss=MAELossConfig(name='mae', reduction='mean'),\n",
+       "                loss_coefficient=1.0,\n",
+       "                type='stresses',\n",
+       "                conservative=False\n",
+       "            )\n",
+       "        ],\n",
+       "        optimizer=AdamWConfig(\n",
+       "            name='AdamW',\n",
+       "            lr=0.0001,\n",
+       "            eps=1e-08,\n",
+       "            betas=(0.9, 0.999),\n",
+       "            weight_decay=0.01,\n",
+       "            amsgrad=False\n",
+       "        ),\n",
+       "        lr_scheduler=None,\n",
+       "        ignore_gpu_batch_transform_error=True,\n",
+       "        name='eqV2',\n",
+       "        checkpoint_path=CachedPath(\n",
+       "            uri='hf://fairchem/OMAT24/eqV2_31M_mp.pt',\n",
+       "            cache_dir=None,\n",
+       "            extract_archive=False,\n",
+       "            force_extract=False,\n",
+       "            quiet=False,\n",
+       "            is_local=False\n",
+       "        ),\n",
+       "        atoms_to_graph=FAIRChemAtomsToGraphSystemConfig(radius=8.0, max_num_neighbors=20)\n",
+       "    ),\n",
+       "    lightning_trainer_kwargs={'fast_dev_run': True, 'accelerator': 'cpu'}\n",
+       ")\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;35mMatterTunerConfig\u001b[0m\u001b[1m(\u001b[0m\n", + " \u001b[33mdata\u001b[0m=\u001b[1;35mPerSplitDataConfig\u001b[0m\u001b[1m(\u001b[0m\n", + " \u001b[33mtrain\u001b[0m=\u001b[1;35mOMAT24DatasetConfig\u001b[0m\u001b[1m(\u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'omat24'\u001b[0m, \u001b[33msrc\u001b[0m=\u001b[1;35mPath\u001b[0m\u001b[1m(\u001b[0m\u001b[32m'/mnt/datasets/salex/val'\u001b[0m\u001b[1m)\u001b[0m\u001b[1m)\u001b[0m,\n", + " \u001b[33mvalidation\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", + " \u001b[33mbatch_size\u001b[0m=\u001b[1;36m1\u001b[0m,\n", + " \u001b[33mnum_workers\u001b[0m=\u001b[1;36m0\u001b[0m,\n", + " \u001b[33mpin_memory\u001b[0m=\u001b[3;92mTrue\u001b[0m\n", + " \u001b[1m)\u001b[0m,\n", + " \u001b[33mmodel\u001b[0m=\u001b[1;35mEqV2BackboneConfig\u001b[0m\u001b[1m(\u001b[0m\n", + " \u001b[33mproperties\u001b[0m=\u001b[1m[\u001b[0m\n", + " \u001b[1;35mEnergyPropertyConfig\u001b[0m\u001b[1m(\u001b[0m\n", + " \u001b[33mname\u001b[0m=\u001b[32m'energy'\u001b[0m,\n", + " \u001b[33mdtype\u001b[0m=\u001b[32m'float'\u001b[0m,\n", + " \u001b[33mloss\u001b[0m=\u001b[1;35mMAELossConfig\u001b[0m\u001b[1m(\u001b[0m\u001b[33mname\u001b[0m=\u001b[32m'mae'\u001b[0m, \u001b[33mreduction\u001b[0m=\u001b[32m'mean'\u001b[0m\u001b[1m)\u001b[0m,\n", + " \u001b[33mloss_coefficient\u001b[0m=\u001b[1;36m1\u001b[0m\u001b[1;36m.0\u001b[0m,\n", + " \u001b[33mtype\u001b[0m=\u001b[32m'energy'\u001b[0m\n", + " \u001b[1m)\u001b[0m,\n", + " \u001b[1;35mForcesPropertyConfig\u001b[0m\u001b[1m(\u001b[0m\n", + " \u001b[33mname\u001b[0m=\u001b[32m'forces'\u001b[0m,\n", + " \u001b[33mdtype\u001b[0m=\u001b[32m'float'\u001b[0m,\n", + " \u001b[33mloss\u001b[0m=\u001b[1;35mMAELossConfig\u001b[0m\u001b[1m(\u001b[0m\u001b[33mname\u001b[0m=\u001b[32m'mae'\u001b[0m, \u001b[33mreduction\u001b[0m=\u001b[32m'mean'\u001b[0m\u001b[1m)\u001b[0m,\n", + " \u001b[33mloss_coefficient\u001b[0m=\u001b[1;36m1\u001b[0m\u001b[1;36m.0\u001b[0m,\n", + " \u001b[33mtype\u001b[0m=\u001b[32m'forces'\u001b[0m,\n", + " \u001b[33mconservative\u001b[0m=\u001b[3;91mFalse\u001b[0m\n", + " \u001b[1m)\u001b[0m,\n", + " \u001b[1;35mStressesPropertyConfig\u001b[0m\u001b[1m(\u001b[0m\n", + " \u001b[33mname\u001b[0m=\u001b[32m'stresses'\u001b[0m,\n", + " \u001b[33mdtype\u001b[0m=\u001b[32m'float'\u001b[0m,\n", + " \u001b[33mloss\u001b[0m=\u001b[1;35mMAELossConfig\u001b[0m\u001b[1m(\u001b[0m\u001b[33mname\u001b[0m=\u001b[32m'mae'\u001b[0m, \u001b[33mreduction\u001b[0m=\u001b[32m'mean'\u001b[0m\u001b[1m)\u001b[0m,\n", + " \u001b[33mloss_coefficient\u001b[0m=\u001b[1;36m1\u001b[0m\u001b[1;36m.0\u001b[0m,\n", + " \u001b[33mtype\u001b[0m=\u001b[32m'stresses'\u001b[0m,\n", + " \u001b[33mconservative\u001b[0m=\u001b[3;91mFalse\u001b[0m\n", + " \u001b[1m)\u001b[0m\n", + " \u001b[1m]\u001b[0m,\n", + " \u001b[33moptimizer\u001b[0m=\u001b[1;35mAdamWConfig\u001b[0m\u001b[1m(\u001b[0m\n", + " \u001b[33mname\u001b[0m=\u001b[32m'AdamW'\u001b[0m,\n", + " \u001b[33mlr\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.0001\u001b[0m,\n", + " \u001b[33meps\u001b[0m=\u001b[1;36m1e\u001b[0m\u001b[1;36m-08\u001b[0m,\n", + " \u001b[33mbetas\u001b[0m=\u001b[1m(\u001b[0m\u001b[1;36m0.9\u001b[0m, \u001b[1;36m0.999\u001b[0m\u001b[1m)\u001b[0m,\n", + " \u001b[33mweight_decay\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.01\u001b[0m,\n", + " \u001b[33mamsgrad\u001b[0m=\u001b[3;91mFalse\u001b[0m\n", + " \u001b[1m)\u001b[0m,\n", + " \u001b[33mlr_scheduler\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", + " \u001b[33mignore_gpu_batch_transform_error\u001b[0m=\u001b[3;92mTrue\u001b[0m,\n", + " \u001b[33mname\u001b[0m=\u001b[32m'eqV2'\u001b[0m,\n", + " \u001b[33mcheckpoint_path\u001b[0m=\u001b[1;35mCachedPath\u001b[0m\u001b[1m(\u001b[0m\n", + " \u001b[33muri\u001b[0m=\u001b[32m'hf://fairchem/OMAT24/eqV2_31M_mp.pt'\u001b[0m,\n", + " \u001b[33mcache_dir\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", + " \u001b[33mextract_archive\u001b[0m=\u001b[3;91mFalse\u001b[0m,\n", + " \u001b[33mforce_extract\u001b[0m=\u001b[3;91mFalse\u001b[0m,\n", + " \u001b[33mquiet\u001b[0m=\u001b[3;91mFalse\u001b[0m,\n", + " \u001b[33mis_local\u001b[0m=\u001b[3;91mFalse\u001b[0m\n", + " \u001b[1m)\u001b[0m,\n", + " \u001b[33matoms_to_graph\u001b[0m=\u001b[1;35mFAIRChemAtomsToGraphSystemConfig\u001b[0m\u001b[1m(\u001b[0m\u001b[33mradius\u001b[0m=\u001b[1;36m8\u001b[0m\u001b[1;36m.0\u001b[0m, \u001b[33mmax_num_neighbors\u001b[0m=\u001b[1;36m20\u001b[0m\u001b[1m)\u001b[0m\n", + " \u001b[1m)\u001b[0m,\n", + " \u001b[33mlightning_trainer_kwargs\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'fast_dev_run'\u001b[0m: \u001b[3;92mTrue\u001b[0m, \u001b[32m'accelerator'\u001b[0m: \u001b[32m'cpu'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[1m)\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): huggingface.co:443\n", + "DEBUG:urllib3.connectionpool:https://huggingface.co:443 \"HEAD /fairchem/OMAT24/resolve/main/eqV2_31M_mp.pt HTTP/11\" 302 0\n", + "DEBUG:git.cmd:Popen(['git', 'version'], cwd=/workspaces/MatterTune/examples, stdin=None, shell=False, universal_newlines=False)\n", + "DEBUG:git.cmd:Popen(['git', 'version'], cwd=/workspaces/MatterTune/examples, stdin=None, shell=False, universal_newlines=False)\n", + "DEBUG:wandb.docker.auth:Trying paths: ['/root/.docker/config.json', '/root/.dockercfg']\n", + "DEBUG:wandb.docker.auth:Found file at path: /root/.docker/config.json\n", + "DEBUG:wandb.docker.auth:Found 'auths' section\n", + "DEBUG:wandb.docker.auth:Auth data for https://index.docker.io/v1/ is absent. Client might be using a credentials store instead.\n", + "DEBUG:wandb.docker.auth:Auth data for registry.heroku.com is absent. Client might be using a credentials store instead.\n", + "DEBUG:wandb.docker.auth:Found 'credsStore' section\n", + "INFO:root:amp: true\n", + "cmd:\n", + " checkpoint_dir: /workspaces/MatterTune/examples/checkpoints/2024-11-06-14-56-48\n", + " commit: null\n", + " identifier: ''\n", + " logs_dir: /workspaces/MatterTune/examples/logs/wandb/2024-11-06-14-56-48\n", + " print_every: 100\n", + " results_dir: /workspaces/MatterTune/examples/results/2024-11-06-14-56-48\n", + " seed: null\n", + " timestamp_id: 2024-11-06-14-56-48\n", + " version: 1.2.1.dev154+g5083262a\n", + "dataset:\n", + " a2g_args:\n", + " r_data_keys:\n", + " - uncorrected_total_energy\n", + " r_forces: true\n", + " r_stress: true\n", + " format: ase_db\n", + " key_mapping:\n", + " uncorrected_total_energy: energy\n", + " transforms:\n", + " decompose_tensor:\n", + " decomposition:\n", + " stress_anisotropic:\n", + " irrep_dim: 2\n", + " stress_isotropic:\n", + " irrep_dim: 0\n", + " rank: 2\n", + " tensor: stress\n", + " element_references:\n", + " file: /private/home/lbluque/checkpoint/mptrj/2024-02-05-17-47/norms-refs-uncorrected/element_references.pt\n", + " normalizer:\n", + " file: /private/home/lbluque/checkpoint/mptrj/2024-02-05-17-47/norms-refs-uncorrected/normalizers.pt\n", + "evaluation_metrics:\n", + " metrics:\n", + " energy:\n", + " - mae\n", + " - mae_density\n", + " forces:\n", + " - mae\n", + " - forcesx_mae\n", + " - forcesy_mae\n", + " - forcesz_mae\n", + " - cosine_similarity\n", + " stress:\n", + " - mae\n", + " - mae_density\n", + " stress_anisotropic:\n", + " - mae\n", + " stress_isotropic:\n", + " - mae\n", + " primary_metric: energy_mae\n", + "gp_gpus: null\n", + "gpus: 0\n", + "logger: wandb\n", + "loss_functions:\n", + "- energy:\n", + " coefficient: 20\n", + " fn: mae_density\n", + "- forces:\n", + " coefficient: 20\n", + " fn: l2mae\n", + "- stress_isotropic:\n", + " coefficient: 5\n", + " fn: mae\n", + "- stress_anisotropic:\n", + " coefficient: 5\n", + " fn: mae\n", + " reduction: mean_all\n", + "model:\n", + " backbone:\n", + " alpha_drop: 0.1\n", + " attn_activation: silu\n", + " attn_alpha_channels: 64\n", + " attn_hidden_channels: 64\n", + " attn_value_channels: 16\n", + " avg_degree: 61.94676351484548\n", + " avg_num_nodes: 31.16592360068011\n", + " distance_function: gaussian\n", + " drop_path_rate: 0.1\n", + " edge_channels: 128\n", + " enforce_max_neighbors_strictly: false\n", + " ffn_activation: silu\n", + " ffn_hidden_channels: 128\n", + " grid_resolution: 18\n", + " lmax_list:\n", + " - 4\n", + " max_neighbors: 20\n", + " max_num_elements: 96\n", + " max_radius: 12.0\n", + " mmax_list:\n", + " - 2\n", + " model: equiformer_v2_backbone\n", + " norm_type: layer_norm_sh\n", + " num_distance_basis: 512\n", + " num_heads: 8\n", + " num_layers: 8\n", + " num_sphere_samples: 128\n", + " otf_graph: true\n", + " proj_drop: 0.0\n", + " share_atom_edge_embedding: false\n", + " sphere_channels: 128\n", + " use_atom_edge_embedding: true\n", + " use_attn_renorm: true\n", + " use_gate_act: false\n", + " use_grid_mlp: true\n", + " use_m_share_rad: false\n", + " use_pbc: true\n", + " use_pbc_single: true\n", + " use_s2_act_attn: false\n", + " use_sep_s2_act: true\n", + " weight_init: uniform\n", + " heads:\n", + " energy:\n", + " module: equiformer_v2_energy_head\n", + " forces:\n", + " module: equiformer_v2_force_head\n", + " stress:\n", + " decompose: true\n", + " module: rank2_symmetric_head\n", + " output_name: stress\n", + " use_source_target_embedding: true\n", + " name: hydra\n", + " otf_graph: true\n", + " pass_through_head_outputs: true\n", + "optim:\n", + " batch_size: 4\n", + " clip_grad_norm: 100\n", + " ema_decay: 0.999\n", + " eval_batch_size: 4\n", + " eval_every: 3000\n", + " load_balancing: atoms\n", + " lr_initial: 0.0002\n", + " max_epochs: 150\n", + " num_workers: 8\n", + " optimizer: AdamW\n", + " optimizer_params:\n", + " weight_decay: 0.001\n", + " scheduler: LambdaLR\n", + " scheduler_params:\n", + " epochs: 416850\n", + " lambda_type: cosine\n", + " lr: 0.0002\n", + " lr_min_factor: 0.01\n", + " warmup_epochs: 277\n", + " warmup_factor: 0.2\n", + "outputs:\n", + " energy:\n", + " level: system\n", + " property: energy\n", + " forces:\n", + " eval_on_free_atoms: true\n", + " level: atom\n", + " property: forces\n", + " train_on_free_atoms: true\n", + " stress:\n", + " decomposition:\n", + " stress_anisotropic:\n", + " eval_on_free_atoms: true\n", + " irrep_dim: 2\n", + " level: system\n", + " parent: stress\n", + " train_on_free_atoms: true\n", + " stress_isotropic:\n", + " eval_on_free_atoms: true\n", + " irrep_dim: 0\n", + " level: system\n", + " parent: stress\n", + " train_on_free_atoms: true\n", + " level: system\n", + " property: stress\n", + "relax_dataset: {}\n", + "slurm:\n", + " additional_parameters:\n", + " constraint: volta32gb\n", + " cpus_per_task: 9\n", + " folder: /private/home/xiangfu/fairchem/src/fairchem/experimental/xiangfu/md/slurm_scripts/logs/33506272\n", + " gpus_per_node: 8\n", + " job_id: '33506272'\n", + " job_name: eqv2_31M_MPTrj_UE_scratch_512globalBS_stress5_150Ep\n", + " mem: 480GB\n", + " nodes: 16\n", + " ntasks_per_node: 8\n", + " partition: learnaccel\n", + " time: 4320\n", + "task: {}\n", + "test_dataset: {}\n", + "trainer: ocp\n", + "val_dataset: {}\n", + "\n", + "INFO:root:Loading model: hydra\n", + "INFO:root:Loaded HydraModel with 31207434 parameters.\n", + "INFO:root:Loading checkpoint in inference-only mode, not loading keys associated with trainer state!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaa_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaa_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_004.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_005.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_006.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_007.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_008.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_009.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_010.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_011.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_012.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_013.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_014.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_015.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_016.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_017.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_018.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_019.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_020.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_021.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_022.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_023.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_024.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_025.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_026.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_027.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_028.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_029.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_030.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_031.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_032.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_033.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_034.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_035.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_036.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_037.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_038.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_039.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_040.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_041.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_042.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_043.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_044.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_045.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_046.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_047.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_048.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_049.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_050.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_051.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_052.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_053.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_054.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_055.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_056.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_057.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_058.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_059.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_060.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_061.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_062.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_063.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_064.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_065.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aab_066.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_004.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_005.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_006.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_007.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_008.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_009.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_010.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_011.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_012.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_013.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_014.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_015.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_016.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_017.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_018.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_019.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_020.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_021.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_022.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_023.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_024.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_025.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_026.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_027.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_028.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_029.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_030.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_031.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_032.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_033.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_034.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_035.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_036.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_037.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_038.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_039.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_040.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_041.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_042.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_043.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_044.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_045.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_046.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_047.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_048.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aac_049.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_004.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_005.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_006.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_007.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_008.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_009.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_010.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_011.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_012.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_013.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_014.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_015.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_016.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_017.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_018.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_019.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_020.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_021.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_022.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_023.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_024.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_025.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_026.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_027.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_028.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_029.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_030.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_031.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_032.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_033.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_034.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_035.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_036.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_037.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_038.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_039.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_040.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_041.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_042.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_043.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_044.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_045.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_046.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_047.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_048.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_049.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_050.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_051.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_052.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_053.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_054.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_055.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_056.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_057.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aad_058.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_004.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_005.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_006.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_007.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_008.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_009.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_010.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_011.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_012.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_013.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_014.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_015.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_016.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_017.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_018.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_019.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_020.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_021.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_022.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_023.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_024.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_025.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_026.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_027.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_028.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_029.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_030.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_031.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_032.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_033.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_034.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_035.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_036.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_037.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_038.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_039.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_040.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_041.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_042.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_043.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_044.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_045.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_046.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_047.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aae_048.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaf_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaf_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaf_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaf_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaf_004.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaf_005.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaf_006.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaf_007.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaf_008.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_004.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_005.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_006.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_007.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_008.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_009.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_010.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_011.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_012.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_013.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_014.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_015.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_016.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_017.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_018.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_019.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_020.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_021.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_022.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_023.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_024.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_025.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_026.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_027.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_028.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_029.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_030.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_031.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_032.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_033.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_034.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_035.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_036.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_037.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_038.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_039.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_040.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_041.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_042.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_043.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_044.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_045.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_046.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_047.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_048.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_049.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_050.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_051.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_052.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_053.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_054.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_055.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_056.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_057.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_058.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_059.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_060.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_061.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_062.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_063.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_064.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_065.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_066.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_067.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_068.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_069.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_070.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_071.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_072.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_073.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_074.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_075.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_076.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_077.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_078.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aag_079.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aah_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aah_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aah_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aah_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aai_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aai_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aai_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aai_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aai_004.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aai_005.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aaj_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aak_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aak_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aak_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aak_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aak_004.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aak_005.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aak_006.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aak_007.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aak_008.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aal_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aam_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aan_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_000.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_001.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_002.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_003.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_004.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_005.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_006.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_007.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_008.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_009.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/alex_go_aao_010.aselmdb-lock but it's not an ASE database!\n", + "DEBUG:root:Tried to connect to /mnt/datasets/salex/val/metadata.npz but it's not an ASE database!\n", + "INFO:lightning_fabric.utilities.rank_zero:GPU available: True (cuda), used: False\n", + "INFO:lightning_fabric.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n", + "INFO:lightning_fabric.utilities.rank_zero:IPU available: False, using: 0 IPUs\n", + "INFO:lightning_fabric.utilities.rank_zero:HPU available: False, using: 0 HPUs\n", + "/opt/conda/envs/jmp-peft/lib/python3.11/site-packages/lightning/pytorch/trainer/setup.py:187: GPU available but not used. You can set it by doing `Trainer(accelerator='gpu')`.\n", + "INFO:lightning_fabric.utilities.rank_zero:Running in `fast_dev_run` mode: will run the requested loop using 1 batch(es). Logging and checkpointing is suppressed.\n", + "/opt/conda/envs/jmp-peft/lib/python3.11/site-packages/lightning/pytorch/trainer/configuration_validator.py:74: You defined a `validation_step` but have no `val_dataloader`. Skipping val loop.\n", + "/opt/conda/envs/jmp-peft/lib/python3.11/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:653: Checkpoint directory /workspaces/MatterTune/examples/checkpoints exists and is not empty.\n", + "INFO: \n", + " | Name | Type | Params\n", + "-------------------------------------------------------\n", + "0 | backbone | EquiformerV2Backbone | 27.8 M\n", + "1 | output_heads | ModuleDict | 3.4 M \n", + "2 | train_metrics | FinetuneMetrics | 0 \n", + "3 | val_metrics | FinetuneMetrics | 0 \n", + "4 | test_metrics | FinetuneMetrics | 0 \n", + "-------------------------------------------------------\n", + "31.2 M Trainable params\n", + "0 Non-trainable params\n", + "31.2 M Total params\n", + "124.830 Total estimated model params size (MB)\n", + "INFO:lightning.pytorch.callbacks.model_summary:\n", + " | Name | Type | Params\n", + "-------------------------------------------------------\n", + "0 | backbone | EquiformerV2Backbone | 27.8 M\n", + "1 | output_heads | ModuleDict | 3.4 M \n", + "2 | train_metrics | FinetuneMetrics | 0 \n", + "3 | val_metrics | FinetuneMetrics | 0 \n", + "4 | test_metrics | FinetuneMetrics | 0 \n", + "-------------------------------------------------------\n", + "31.2 M Trainable params\n", + "0 Non-trainable params\n", + "31.2 M Total params\n", + "124.830 Total estimated model params size (MB)\n", + "/opt/conda/envs/jmp-peft/lib/python3.11/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:441: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=31` in the `DataLoader` to improve performance.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "13d11808e84b46f9b3dc55bfbf444408", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Training: | | 0/? [00:00\n", + "\n" + ] + } + ], + "source": [ + "potential = model.potential(hp.lightning_trainer_kwargs)\n", + "print(potential)\n", + "\n", + "calculator = model.ase_calculator(hp.lightning_trainer_kwargs)\n", + "print(calculator)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:lightning_fabric.utilities.rank_zero:GPU available: True (cuda), used: False\n", + "INFO:lightning_fabric.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n", + "INFO:lightning_fabric.utilities.rank_zero:IPU available: False, using: 0 IPUs\n", + "INFO:lightning_fabric.utilities.rank_zero:HPU available: False, using: 0 HPUs\n", + "INFO:lightning_fabric.utilities.rank_zero:Running in `fast_dev_run` mode: will run the requested loop using 1 batch(es). Logging and checkpointing is suppressed.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Atoms(symbols='H2O', pbc=True, cell=[10.0, 10.0, 10.0])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/envs/jmp-peft/lib/python3.11/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:441: The 'predict_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=31` in the `DataLoader` to improve performance.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "82bfc48c1dcc4237a8ca19233f32f6e9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Predicting: | | 0/? [00:00 nn.Module: + from fairchem.core.common.registry import registry + from fairchem.core.common.utils import update_config + + if isinstance(checkpoint_path := hparams.checkpoint_path, CE.CachedPath): + checkpoint_path = checkpoint_path.resolve() + + checkpoint = None + # Loads the config from the checkpoint directly (always on CPU). + checkpoint = torch.load(checkpoint_path, map_location=torch.device("cpu")) + config = checkpoint["config"] + + config["trainer"] = config.get("trainer", "ocp") + + if "model_attributes" in config: + config["model_attributes"]["name"] = config.pop("model") + config["model"] = config["model_attributes"] + + # Calculate the edge indices on the fly + config["model"]["otf_graph"] = True + + ### backwards compatability with OCP v<2.0 + config = update_config(config) + + # Save config so obj can be transported over network (pkl) + config["checkpoint"] = checkpoint_path + del config["dataset"]["src"] + + # Import a bunch of modules so that the registry can find the classes + import fairchem.core.models # noqa: F401 + import fairchem.core.models.equiformer_v2 + import fairchem.core.models.equiformer_v2.equiformer_v2 + import fairchem.core.models.equiformer_v2.prediction_heads.rank2 + import fairchem.core.trainers + + trainer = registry.get_trainer_class(config["trainer"])( + task=config.get("task", {}), + model=config["model"], + dataset=[config["dataset"]], + outputs=config["outputs"], + loss_functions=config["loss_functions"], + evaluation_metrics=config["evaluation_metrics"], + optimizer=config["optim"], + identifier="", + slurm=config.get("slurm", {}), + local_rank=config.get("local_rank", 0), + is_debug=config.get("is_debug", True), + cpu=True, + amp=config.get("amp", False), + inference_only=True, + ) + + # Load the checkpoint + if checkpoint_path is not None: + try: + trainer.load_checkpoint(checkpoint_path, checkpoint, inference_only=True) + except NotImplementedError: + log.warning(f"Unable to load checkpoint from {checkpoint_path}") + + # Now, extract the backbone from the trainer and delete the trainer + from fairchem.core.trainers import OCPTrainer + + assert isinstance(trainer, OCPTrainer), "Only OCPTrainer is supported." + assert (model := getattr(trainer, "_unwrapped_model", None)) is not None, ( + "The model could not be extracted from the trainer. " + "Please report this issue." + ) + + # Make sure this is eqv2 + from fairchem.core.models.base import HydraModel + + assert isinstance( + model, HydraModel + ), f"Expected model to be of type HydraModel, but got {type(model)}" + + from fairchem.core.models.equiformer_v2.equiformer_v2 import EquiformerV2Backbone + + assert isinstance( + backbone := model.backbone, EquiformerV2Backbone + ), f"Expected backbone to be of type EquiformerV2Backbone, but got {type(backbone)}" + + return backbone + + +class EqV2BackboneModule(FinetuneModuleBase["BaseData", "Batch", EqV2BackboneConfig]): + @override + @classmethod + def hparams_cls(cls): + return EqV2BackboneConfig + + @override + @classmethod + def ensure_dependencies(cls): + # Make sure the fairchem module is available + if importlib.util.find_spec("fairchem") is None: + raise ImportError( + "The fairchem module is not installed. Please install it by running" + " pip install fairchem-core." + ) + + # Make sure torch-geometric is available + if importlib.util.find_spec("torch_geometric") is None: + raise ImportError( + "The torch-geometric module is not installed. Please install it by running" + " pip install torch-geometric." + ) + + @override + def requires_disabled_inference_mode(self): + return False + + def _create_output_head(self, prop: props.PropertyConfig): + match prop: + case props.EnergyPropertyConfig(): + from fairchem.core.models.equiformer_v2.equiformer_v2 import ( + EquiformerV2EnergyHead, + ) + + return EquiformerV2EnergyHead(self.backbone, reduce="sum") + case props.ForcesPropertyConfig(): + assert ( + not prop.conservative + ), "Conservative forces are not supported for eqV2 (yet)" + + from fairchem.core.models.equiformer_v2.equiformer_v2 import ( + EquiformerV2ForceHead, + ) + + return EquiformerV2ForceHead(self.backbone) + case props.StressesPropertyConfig(): + assert ( + not prop.conservative + ), "Conservative stresses are not supported for eqV2 (yet)" + + from fairchem.core.models.equiformer_v2.prediction_heads.rank2 import ( + Rank2SymmetricTensorHead, + ) + + return Rank2SymmetricTensorHead( + self.backbone, + output_name="stress", + use_source_target_embedding=True, + decompose=True, + extensive=False, + ) + case props.GraphPropertyConfig(): + assert prop.reduction in ("sum", "mean"), ( + f"Unsupported reduction: {prop.reduction} for eqV2. " + "Please use 'sum' or 'mean'." + ) + from fairchem.core.models.equiformer_v2.equiformer_v2 import ( + EquiformerV2EnergyHead, + ) + + return EquiformerV2EnergyHead(self.backbone, reduce=prop.reduction) + case _: + raise ValueError( + f"Unsupported property config: {prop} for eqV2" + "Please ask the maintainers of eqV2 for support" + ) + + @override + def create_model(self): + # Get the pre-trained backbone + self.backbone = _get_backbone(self.hparams) + + # Create the output heads + self.output_heads = nn.ModuleDict() + for prop in self.hparams.properties: + self.output_heads[prop.name] = self._create_output_head(prop) + + @override + @contextlib.contextmanager + def model_forward_context(self, data): + yield + + @override + def model_forward(self, batch, return_backbone_output=False): + # Run the backbone + emb = self.backbone(batch) + + # Feed the backbone output to the output heads + predicted_properties: dict[str, torch.Tensor] = {} + for name, head in self.output_heads.items(): + assert ( + prop := next( + (p for p in self.hparams.properties if p.name == name), None + ) + ) is not None, ( + f"Property {name} not found in properties. " + "This should not happen, please report this." + ) + + head_output: dict[str, torch.Tensor] = head(batch, emb) + + match prop: + case props.EnergyPropertyConfig(): + pred = head_output["energy"] + case props.ForcesPropertyConfig(): + pred = head_output["forces"] + case props.StressesPropertyConfig(): + # Convert the stress tensor to the full 3x3 form + stress_rank0 = head_output["stress_isotropic"] # (bsz 1) + stress_rank2 = head_output["stress_anisotropic"] # (bsz, 5) + stress = torch.cat([stress_rank0, stress_rank2], dim=1) # (bsz, 6) + pred = voigt_6_to_full_3x3_stress_torch(stress) + case props.GraphPropertyConfig(): + pred = head_output["energy"] + case _: + assert_never(prop) + + predicted_properties[name] = pred + + pred_dict: ModelOutput = {"predicted_properties": predicted_properties} + if return_backbone_output: + pred_dict["backbone_output"] = emb + + return pred_dict + + @override + def pretrained_backbone_parameters(self): + return self.backbone.parameters() + + @override + def output_head_parameters(self): + for head in self.output_heads.values(): + yield from head.parameters() + + @override + def cpu_data_transform(self, data): + return data + + @override + def collate_fn(self, data_list): + from fairchem.core.datasets import data_list_collater + + return cast("Batch", data_list_collater(data_list, otf_graph=True)) + + @override + def gpu_batch_transform(self, batch): + return batch + + @override + def batch_to_labels(self, batch): + HARDCODED_NAMES: dict[type[props.PropertyConfigBase], str] = { + props.EnergyPropertyConfig: "energy", + props.ForcesPropertyConfig: "forces", + props.StressesPropertyConfig: "stress", + } + + labels: dict[str, torch.Tensor] = {} + for prop in self.hparams.properties: + batch_prop_name = HARDCODED_NAMES.get(type(prop), prop.name) + labels[prop.name] = batch[batch_prop_name] + + return labels + + @override + def atoms_to_data(self, atoms, has_labels): + from fairchem.core.preprocessing import AtomsToGraphs + + energy = False + forces = False + stress = False + data_keys = None + if has_labels: + energy = any( + isinstance(prop, props.EnergyPropertyConfig) + for prop in self.hparams.properties + ) + forces = any( + isinstance(prop, props.ForcesPropertyConfig) + for prop in self.hparams.properties + ) + stress = any( + isinstance(prop, props.StressesPropertyConfig) + for prop in self.hparams.properties + ) + data_keys = [ + prop.name + for prop in self.hparams.properties + if not isinstance( + prop, + ( + props.EnergyPropertyConfig, + props.ForcesPropertyConfig, + props.StressesPropertyConfig, + ), + ) + ] + + a2g = AtomsToGraphs( + max_neigh=self.hparams.atoms_to_graph.max_num_neighbors, + radius=self.hparams.atoms_to_graph.radius, + r_energy=energy, + r_forces=forces, + r_stress=stress, + r_data_keys=data_keys, + r_distances=False, + r_edges=False, + r_pbc=True, + ) + data = a2g.convert(atoms) + + # Reshape the cell and stress tensors to (1, 3, 3) + # so that they can be properly batched by the collate_fn. + if hasattr(data, "cell"): + data.cell = data.cell.reshape(1, 3, 3) + if hasattr(data, "stress"): + data.stress = data.stress.reshape(1, 3, 3) + + return data diff --git a/src/mattertune/backbones/orb/model.py b/src/mattertune/backbones/orb/model.py index 06ede38..95de581 100644 --- a/src/mattertune/backbones/orb/model.py +++ b/src/mattertune/backbones/orb/model.py @@ -13,6 +13,7 @@ from ...finetune import properties as props from ...finetune.base import FinetuneModuleBase, FinetuneModuleBaseConfig, ModelOutput from ...registry import backbone_registry +from ..util import voigt_6_to_full_3x3_stress_torch if TYPE_CHECKING: from orb_models.forcefield.base import AtomGraphs @@ -21,33 +22,6 @@ log = logging.getLogger(__name__) -def _voigt_6_to_full_3x3_stress_torch(stress_vector: torch.Tensor) -> torch.Tensor: - """ - Form a 3x3 stress matrix from a 6 component vector in Voigt notation - - Args: - stress_vector: Tensor of shape (B, 6) where B is the batch size - - Returns: - Tensor of shape (B, 3, 3) - """ - # Unpack the components - s1, s2, s3, s4, s5, s6 = stress_vector.unbind(dim=1) - - # Stack the components into a 3x3 matrix - # Each s_i is of shape (B,) - stress_matrix = torch.stack( - [ - torch.stack([s1, s6, s5], dim=1), - torch.stack([s6, s2, s4], dim=1), - torch.stack([s5, s4, s3], dim=1), - ], - dim=1, - ) - - return stress_matrix - - class ORBSystemConfig(C.Config): """Config controlling how to featurize a system of atoms.""" @@ -255,7 +229,7 @@ def model_forward(self, batch, return_backbone_output=False): # Convert the stress tensor to the full 3x3 form if isinstance(prop, props.StressesPropertyConfig): - pred = _voigt_6_to_full_3x3_stress_torch(pred) + pred = voigt_6_to_full_3x3_stress_torch(pred) predicted_properties[name] = pred diff --git a/src/mattertune/backbones/util.py b/src/mattertune/backbones/util.py new file mode 100644 index 0000000..36efded --- /dev/null +++ b/src/mattertune/backbones/util.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import torch + + +def voigt_6_to_full_3x3_stress_torch(stress_vector: torch.Tensor) -> torch.Tensor: + """ + Form a 3x3 stress matrix from a 6 component vector in Voigt notation + + Args: + stress_vector: Tensor of shape (B, 6) where B is the batch size + + Returns: + Tensor of shape (B, 3, 3) + """ + # Unpack the components + s1, s2, s3, s4, s5, s6 = stress_vector.unbind(dim=1) + + # Stack the components into a 3x3 matrix + # Each s_i is of shape (B,) + stress_matrix = torch.stack( + [ + torch.stack([s1, s6, s5], dim=1), + torch.stack([s6, s2, s4], dim=1), + torch.stack([s5, s4, s3], dim=1), + ], + dim=1, + ) + + return stress_matrix diff --git a/src/mattertune/configs/__init__.py b/src/mattertune/configs/__init__.py index 1b49c9f..dadb492 100644 --- a/src/mattertune/configs/__init__.py +++ b/src/mattertune/configs/__init__.py @@ -5,11 +5,18 @@ from mattertune.backbones import JMPBackboneConfig as JMPBackboneConfig from mattertune.backbones import M3GNetBackboneConfig as M3GNetBackboneConfig from mattertune.backbones import ModelConfig as ModelConfig +from mattertune.backbones.eqV2 import EqV2BackboneConfig as EqV2BackboneConfig +from mattertune.backbones.eqV2.model import ( + FAIRChemAtomsToGraphSystemConfig as FAIRChemAtomsToGraphSystemConfig, +) from mattertune.backbones.jmp.model import CutoffsConfig as CutoffsConfig from mattertune.backbones.jmp.model import ( JMPGraphComputerConfig as JMPGraphComputerConfig, ) from mattertune.backbones.jmp.model import MaxNeighborsConfig as MaxNeighborsConfig +from mattertune.backbones.jmp.prediction_heads.graph_scalar import ( + GraphScalarTargetConfig as GraphScalarTargetConfig, +) from mattertune.backbones.m3gnet import ( M3GNetGraphComputerConfig as M3GNetGraphComputerConfig, ) diff --git a/src/mattertune/configs/backbones/__init__.py b/src/mattertune/configs/backbones/__init__.py index 7e9c52c..87d92a3 100644 --- a/src/mattertune/configs/backbones/__init__.py +++ b/src/mattertune/configs/backbones/__init__.py @@ -6,17 +6,25 @@ from mattertune.backbones import JMPBackboneConfig as JMPBackboneConfig from mattertune.backbones import M3GNetBackboneConfig as M3GNetBackboneConfig from mattertune.backbones import ModelConfig as ModelConfig +from mattertune.backbones.eqV2 import EqV2BackboneConfig as EqV2BackboneConfig +from mattertune.backbones.eqV2.model import ( + FAIRChemAtomsToGraphSystemConfig as FAIRChemAtomsToGraphSystemConfig, +) from mattertune.backbones.jmp.model import CutoffsConfig as CutoffsConfig from mattertune.backbones.jmp.model import ( JMPGraphComputerConfig as JMPGraphComputerConfig, ) from mattertune.backbones.jmp.model import MaxNeighborsConfig as MaxNeighborsConfig +from mattertune.backbones.jmp.prediction_heads.graph_scalar import ( + GraphScalarTargetConfig as GraphScalarTargetConfig, +) from mattertune.backbones.m3gnet import ( M3GNetGraphComputerConfig as M3GNetGraphComputerConfig, ) from mattertune.backbones.orb import ORBBackboneConfig as ORBBackboneConfig from mattertune.backbones.orb.model import ORBSystemConfig as ORBSystemConfig +from . import eqV2 as eqV2 from . import jmp as jmp from . import m3gnet as m3gnet from . import orb as orb diff --git a/src/mattertune/configs/backbones/eqV2/__init__.py b/src/mattertune/configs/backbones/eqV2/__init__.py new file mode 100644 index 0000000..9f52c3c --- /dev/null +++ b/src/mattertune/configs/backbones/eqV2/__init__.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +__codegen__ = True + +from mattertune.backbones.eqV2 import EqV2BackboneConfig as EqV2BackboneConfig +from mattertune.backbones.eqV2.model import ( + FAIRChemAtomsToGraphSystemConfig as FAIRChemAtomsToGraphSystemConfig, +) +from mattertune.backbones.eqV2.model import ( + FinetuneModuleBaseConfig as FinetuneModuleBaseConfig, +) + +from . import model as model diff --git a/src/mattertune/configs/backbones/eqV2/model/__init__.py b/src/mattertune/configs/backbones/eqV2/model/__init__.py new file mode 100644 index 0000000..e99b2fd --- /dev/null +++ b/src/mattertune/configs/backbones/eqV2/model/__init__.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +__codegen__ = True + +from mattertune.backbones.eqV2.model import EqV2BackboneConfig as EqV2BackboneConfig +from mattertune.backbones.eqV2.model import ( + FAIRChemAtomsToGraphSystemConfig as FAIRChemAtomsToGraphSystemConfig, +) +from mattertune.backbones.eqV2.model import ( + FinetuneModuleBaseConfig as FinetuneModuleBaseConfig, +) diff --git a/src/mattertune/configs/backbones/jmp/__init__.py b/src/mattertune/configs/backbones/jmp/__init__.py index 97bc088..72120f2 100644 --- a/src/mattertune/configs/backbones/jmp/__init__.py +++ b/src/mattertune/configs/backbones/jmp/__init__.py @@ -11,5 +11,9 @@ JMPGraphComputerConfig as JMPGraphComputerConfig, ) from mattertune.backbones.jmp.model import MaxNeighborsConfig as MaxNeighborsConfig +from mattertune.backbones.jmp.prediction_heads.graph_scalar import ( + GraphScalarTargetConfig as GraphScalarTargetConfig, +) from . import model as model +from . import prediction_heads as prediction_heads diff --git a/src/mattertune/configs/backbones/jmp/prediction_heads/__init__.py b/src/mattertune/configs/backbones/jmp/prediction_heads/__init__.py new file mode 100644 index 0000000..04c2773 --- /dev/null +++ b/src/mattertune/configs/backbones/jmp/prediction_heads/__init__.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +__codegen__ = True + +from mattertune.backbones.jmp.prediction_heads.graph_scalar import ( + GraphScalarTargetConfig as GraphScalarTargetConfig, +) + +from . import graph_scalar as graph_scalar diff --git a/src/mattertune/configs/backbones/jmp/prediction_heads/graph_scalar/__init__.py b/src/mattertune/configs/backbones/jmp/prediction_heads/graph_scalar/__init__.py new file mode 100644 index 0000000..28df014 --- /dev/null +++ b/src/mattertune/configs/backbones/jmp/prediction_heads/graph_scalar/__init__.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +__codegen__ = True + +from mattertune.backbones.jmp.prediction_heads.graph_scalar import ( + GraphScalarTargetConfig as GraphScalarTargetConfig, +)