Well embarassingly after the third time looking at it, I believe the errors were due to my self-sabotage. Perhaps choosing address instead of constant for duration. It is shamefull that I don't know how I fixed it. For anyone interested, here it is:
TSCR1 equ $0046
TSCR2 equ $004D
TIOS equ $0040
TCNT equ $0044
TCTL1 equ $0048
TC5 equ $005A
TIE equ $004C
TC0 equ $0050
TFLG1 equ $004E
OC5 equ $20
UserTimerCh5 equ $3E64
OC4 equ $10
OC0 equ $00
DDRB equ $0003
G3 equ 7653 ; delay count to generate G3 note (with 1:8 prescaler)
B3 equ 6074 ; delay count to generate B3 note (with 1:8 prescaler)
C4 equ 5733 ; delay count to generate C4 note (with 1:8 prescaler)
C4S equ 5412 ; delay count to generate C4S note
D4 equ 5108 ;;;;;;experiment; delay count to generate D4 note (with 1:8 prescaler)
E4 equ 4551 ; delay count to generate E4 note (with 1:8 prescaler)
F4 equ 4295 ; delay count to generate F4 note (with 1:8 prescaler)
F4S equ 4054 ; delay count to generate F4S note (with 1:8 prescaler)
G4 equ 3827 ; delay count to generate G4 note (with 1:8 prescaler)
A4 equ 3409 ; delay count to generate A4 note (with 1:8 prescaler)
B4F equ 3218 ; delay count to generate B4F note (with 1:8 prescaler)
B4 equ 3037 ; delay count to generate B4 note (with 1:8 prescaler)
C5 equ 2867 ; delay count to generate C5 note (with 1:8 prescaler)
D5 equ 2554 ; delay count to generate D5 note (with 1:8 prescaler)
E5 equ 2275 ; delay count to generate E5 note (with 1:8 prescaler)
F5 equ 2148 ; delay count to generate F5 note (with 1:8 prescaler)
ZZ equ 20 ; delay count to generate an inaudible note
notes equ 118
toggle equ $04 ; value to toggle the TC5 pin
org $1000
delay dw 1 ; store the delay for output compare operation
rep_cnt db 1 ; repeat the song this many times
ip db 1 ; remaining notes to be played
org $1500
; establish the SRAM vector address for OC5
lds #$1500
movw #oc5_isr,UserTimerCh5
;movb #$0000,delay
;movb #$FF,DDRB
movb #$90,TSCR1 ; enable TCNT, fast timer flag clear
movb #$03,TSCR2 ; set main timer prescaler to 8
bset TIOS,OC5 ; enable OC5
movb #toggle,TCTL1 ; select toggle for OC5 pin action
ldx #score ; use as a pointer to score table
ldy #duration,delay ; points to duration table
movb #1,rep_cnt ; play the song twice
movb #notes,ip
movw 2,x+,delay ; start with 0th note
ldd TCNT ; play the first note
addd delay ; "
std TC5 ; "
bset TIE,OC5 ; enable OC5 interrupt
cli ; "
forever pshy ; save duration table pointer in stack
ldy 0,y ; get the duration of the current note
jsr delayby10ms ; "
puly ; get the duration pointer from stack
iny ; move the duration pointer
iny ; "
ldd 2,x+ ; get the next note, move pointer
std delay ; "
dec ip
bne forever
dec rep_cnt
beq done ; if not finished playing, re-establish
ldx #score ; pointers and loop count
ldy #duration ; "
movb #notes,ip ; "
movw 0,x,delay ; get the first note delay count
ldd TCNT ; play the first note
addd delay ; "
std TC5
bra forever
done swi
; *******************************************************************************************
; The OC5 interrupt service routine simply starts a new OC5 operation.
; *******************************************************************************************
oc5_isr ldd TC5
addd delay
std TC5
rti
; *******************************************************************************************
; The following subroutine creates a time delay which is equal to [Y] times
; 10 ms. The timer prescaler is 1:8.
; *******************************************************************************************
delayby10ms
pshd
bset TIOS,$01 ; enable OC0
ldd TCNT
again1 addd #30000 ; start an output compare operation
std TC0 ; with 10-ms time delay ;0
brclr tflg1,$01,* ;brclr
ldd TC0 ;0
dbne y,again1
bclr TIOS,$01 ; disable OC0
puld
rts
; ***********************************************************************************************
; This table determines the frequency of each note.
; ***********************************************************************************************
score dw D4,B3,G3,B3,D4,G4,B4,A4,G4,B3,C4S
dw D4,ZZ,D4,ZZ,D4,B4,A4,G4,F4S,E4,F4S,G4,ZZ,G4,D4,B3,G3
dw D4,B3,G3,B3,D4,G4,B4,A4,G4,B3,C4S,D4,ZZ,D4,ZZ,D4
dw B4,A4,G4,F4S,E4,F4S,G4,ZZ,G4,D4,B3,G3,B4,ZZ,B4
dw B4,C5,D5,ZZ,D5,C5,B4,A4,B4,C5,ZZ,C5,ZZ,C5,B4,A4,G4
dw F4S,E4,F4S,G4,B3,C4S,D4,ZZ,D4,G4,ZZ,G4,ZZ,G4,F4S
dw E4,ZZ,E4,ZZ,E4,A4,C5,B4,A4,G4,ZZ,G4,F4S,D4,ZZ,D4
dw G4,A4,B4,C5,D5,G4,A4,B4,C5,A4,G4
; ***********************************************************************************************
; Each of the following entries multiplied by 10 ms gives the duration of a note.
; ***********************************************************************************************
duration
dw 30,10,40,40,40,80,30,10,40,40,40
dw 80, 3,20,3,20,60,20,40,80,20,20,40,3,40,40,40,40
dw 30,10,40,40,40,80,30,10,40,40,40,80,3,20,3,20
dw 60,20,40,80,20,20,40,3,40,40,40,40,20,3,20
dw 40,40,40,3,80,20,20,40,40,40,3,80,3,40,60,20,40
dw 80,20,20,40,40,40,80,3,40,40,3,40,3,20,20
dw 40, 3,40,3,40,40,20,20,20,20,3,40,40,20,3,20
dw 60,20,20,20,80,20,20,60,20,40,80
end