රොටරිස්ථානීය කාවැද්දීම් (කඹය) අත්හදා බැලීම

14from labml import experiment
15from labml.configs import option, calculate
16from labml_nn.transformers import TransformerConfigs
17from labml_nn.transformers.basic.autoregressive_experiment import AutoregressiveTransformer, Configs

21def _rotary_pe_mha(c: TransformerConfigs):
22    from labml_nn.transformers.rope import RotaryPEMultiHeadAttention
23    return RotaryPEMultiHeadAttention(c.n_heads, c.d_model, 1.)

27calculate(TransformerConfigs.encoder_attn, 'rotary', _rotary_pe_mha)
28calculate(TransformerConfigs.decoder_attn, 'rotary', _rotary_pe_mha)
29calculate(TransformerConfigs.decoder_mem_attn, 'rotary', _rotary_pe_mha)

32@option(Configs.model, 'rotary_pe_transformer')
33def _model(c: Configs):

37    m = AutoregressiveTransformer(c.transformer.encoder,
38                                  c.transformer.src_embed,
39                                  c.transformer.generator).to(c.device)
40
41    return m

44def main():

46    experiment.create(name="rotary_pe_transformer", writers={'screen'})

48    conf = Configs()

50    experiment.configs(conf, {

52        'transformer.src_embed': 'no_pos',
53        'transformer.tgt_embed': 'no_pos',

56        'transformer.encoder_attn': 'rotary',

59        'model': 'rotary_pe_transformer',

62        'tokenizer': 'character',

64        'prompt_separator': '',

66        'prompt': 'It is ',

68        'text': 'tiny_shakespeare',

71        'seq_len': 512,

73        'epochs': 32,

75        'batch_size': 4,

78        'inner_iterations': 10,

81        'd_model': 128,
82        'transformer.ffn.d_ff': 512,
83        'transformer.n_heads': 16,
84        'transformer.dropout': 0.0,

87        'optimizer.optimizer': 'Noam',
88        'optimizer.learning_rate': 1.,
89
90        'dataloader_shuffle_with_replacement': True
91    })

94    experiment.add_pytorch_models({'model': conf.model})

97    with experiment.start():

99        conf.run()

103if __name__ == '__main__':
104    main()